Update nlu evaluation with basic intent classification using fuzzy comparison

app.py

@@ -97,8 +97,8 @@ async def evaluate_user_message_with_nlu_api(request: Request):
 
     Output
     - int_data_dict or sent_data_dict: dict - the type of NLU run and result
-      {'type':'integer', 'data': '8'}
-      {'type':'sentiment', 'data': 'negative'}
+      {'type':'integer', 'data': '8', 'confidence': 0}
+      {'type':'sentiment', 'data': 'negative', 'confidence': 0.99}
     """
     data_dict = await request.json()
     message_data = data_dict.get('message_data', '')

mathtext_fastapi/data/text2int_results.csv

@@ -1,92 +1,105 @@
 input,output,text2int,score
-notanumber,32202,32202,True
-this is not a number,32202,32202,True
-fourteen,14,14,True
-forteen,14,14,True
-one thousand four hundred ninety two,1492,1492,True
-one thousand ninety two,1092,1092,True
-Fourteen Hundred Ninety-Two,1492,1492,True
-Fourteen Hundred,1400,1400,True
-Ninety nine,99,99,True
-fifteen thousand five hundred-sixty,15560,15560,True
-three hundred fifty,350,350,True
-one nine eight five,1985,1985,True
-nineteen eighty-five,1985,1605,False
-oh one,1,1,True
-six oh 1,601,601,True
-sex,6,6,True
-six,6,6,True
-eight oh,80,8,False
-eighty,80,80,True
-ate,8,1,False
-double eight,88,32202,False
-eight three seven five three O nine,8375309,8375329,False
-eight three seven five three oh nine,8375309,8375309,True
-eight three seven five three zero nine,8375309,8375309,True
-eight three seven five three oh ni-ee-ine,8375309,837530611,False
-two eight,28,16,False
-seven oh eleven,7011,77,False
-seven elevens,77,77,True
-seven eleven,711,77,False
-ninety nine oh five,9905,149,False
-seven 0 seven 0 seven 0 seven,7070707,7070707,True
-123 hundred,123000,223,False
-5 o 5,505,525,False
-15 o 5,1505,22,False
-15-o 5,1505,22,False
-15 o-5,1505,22,False
-911-thousand,911000,911000,True
-twenty-two twenty-two,2222,44,False
-twenty-two twenty-twos,484,44,False
-four eighty four,484,404,False
-four eighties,320,72,False
-four eighties and nine nineties,1130,243,False
-ninety nine hundred and seventy seven,9977,276,False
-seven thousands,7000,7000,True
-2 hundreds,200,200,True
-99 thousands and one,99001,99001,True
-"forty-five thousand, seven hundred and nine",45709,1161,False
-eighty eight hundred eighty,8880,268,False
-a hundred hundred,10000,32202,False
-a hundred thousand,100000,32202,False
-a hundred million,100000000,32202,False
-nineteen ninety nine,1999,1809,False
-forteen twenty seven,1427,307,False
-seventeen-thousand and seventy two,17072,17072,True
-two hundred and nine,209,209,True
-two thousand ten,2010,2010,True
-two thousand and ten,2010,2010,True
-twelve million,12000000,12000000,True
-8 billion,8000000000,8000000000,True
-twenty ten,2010,2010,True
-thirty-two hundred,3200,3200,True
-nine,9,9,True
-forty two,42,42,True
-1 2 three,123,123,True
-fourtean,14,14,True
-one tousand four hundred ninty two,1492,1492,True
-Furteen Hundrd Ninety-Too,1492,1492,True
-forrteen,14,14,True
-sevnteen-thosand and seventy two,17072,17072,True
-ninety nine hundred ad seventy seven,9977,32202,False
-seven thusands,7000,7000,True
-2 hunreds,200,200,True
-99 tousands and one,99001,99001,True
-eighty ate hundred eighty,8880,261,False
-fourteen Hundred,1400,1400,True
-8 Bilion,8000000000,8000000,False
-one million three thousand one,1003001,1003001,True
-four million nine thousand seven,4009007,4009007,True
-two million five hundred thousand,2500000,2001500,False
-two tousand ten,2010,2010,True
-two thousand teen,2010,2007,False
-tvelve milion,12000000,12000000,True
-tventy ten,2010,2010,True
-tirty-twoo hunred,3200,3200,True
-sevn thoosands,7000,7000,True
-five,5,5,True
-ten,10,10,True
-one two three and ten,12310,51,False
-ONE MILLion three hunded and fiv,1000305,1000305,True
-"50,500 and six",50506,50506,True
-one_million_and_five,1000005,1000005,True
+notanumber,32202.0,32202.0,True
+this is not a number,32202.0,32202.0,True
+fourteen,14.0,14.0,True
+forteen,14.0,14.0,True
+one thousand four hundred ninety two,1492.0,1492.0,True
+one thousand ninety two,1092.0,1092.0,True
+Fourteen Hundred Ninety-Two,1492.0,1492.0,True
+Fourteen Hundred,1400.0,1400.0,True
+Ninety nine,99.0,99.0,True
+fifteen thousand five hundred-sixty,15560.0,15560.0,True
+three hundred fifty,350.0,350.0,True
+one nine eight five,1985.0,1985.0,True
+nineteen eighty-five,1985.0,1605.0,False
+oh one,1.0,1.0,True
+six oh 1,601.0,601.0,True
+sex,6.0,6.0,True
+six,6.0,6.0,True
+eight oh,80.0,8.0,False
+eighty,80.0,80.0,True
+ate,8.0,1.0,False
+double eight,88.0,8.0,False
+eight three seven five three O nine,8375309.0,8375329.0,False
+eight three seven five three oh nine,8375309.0,8375309.0,True
+eight three seven five three zero nine,8375309.0,8375309.0,True
+eight three seven five three oh ni-ee-ine,8375309.0,837530619.0,False
+two eight,28.0,16.0,False
+seven oh eleven,7011.0,77.0,False
+seven elevens,77.0,77.0,True
+seven eleven,711.0,77.0,False
+ninety nine oh five,9905.0,149.0,False
+seven 0 seven 0 seven 0 seven,7070707.0,7070707.0,True
+123 hundred,123000.0,223.0,False
+5 o 5,505.0,525.0,False
+15 o 5,1505.0,22.0,False
+15-o 5,1505.0,22.0,False
+15 o-5,1505.0,22.0,False
+911-thousand,911000.0,911000.0,True
+twenty-two twenty-two,2222.0,44.0,False
+twenty-two twenty-twos,484.0,44.0,False
+four eighty four,484.0,404.0,False
+four eighties,320.0,72.0,False
+four eighties and nine nineties,1130.0,243.0,False
+ninety nine hundred and seventy seven,9977.0,276.0,False
+seven thousands,7000.0,7000.0,True
+2 hundreds,200.0,200.0,True
+99 thousands and one,99001.0,99001.0,True
+"forty-five thousand, seven hundred and nine",45709.0,1161.0,False
+eighty eight hundred eighty,8880.0,268.0,False
+a hundred hundred,10000.0,100.0,False
+a hundred thousand,100000.0,100.0,False
+a hundred million,100000000.0,100.0,False
+nineteen ninety nine,1999.0,1809.0,False
+forteen twenty seven,1427.0,307.0,False
+seventeen-thousand and seventy two,17072.0,17072.0,True
+two hundred and nine,209.0,209.0,True
+two thousand ten,2010.0,2010.0,True
+two thousand and ten,2010.0,2010.0,True
+twelve million,12000000.0,12000000.0,True
+8 billion,8000000000.0,8000000000.0,True
+twenty ten,2010.0,2010.0,True
+thirty-two hundred,3200.0,3200.0,True
+nine,9.0,9.0,True
+forty two,42.0,42.0,True
+1 2 three,123.0,123.0,True
+fourtean,14.0,14.0,True
+one tousand four hundred ninty two,1492.0,1492.0,True
+Furteen Hundrd Ninety-Too,1492.0,1492.0,True
+forrteen,14.0,14.0,True
+sevnteen-thosand and seventy two,17072.0,17072.0,True
+ninety nine hundred ad seventy seven,9977.0,90.0,False
+seven thusands,7000.0,7000.0,True
+2 hunreds,200.0,200.0,True
+99 tousands and one,99001.0,99001.0,True
+eighty ate hundred eighty,8880.0,261.0,False
+fourteen Hundred,1400.0,1400.0,True
+8 Bilion,8000000000.0,8000000.0,False
+one million three thousand one,1003001.0,1003001.0,True
+four million nine thousand seven,4009007.0,4009007.0,True
+two million five hundred thousand,2500000.0,2001500.0,False
+two tousand ten,2010.0,2010.0,True
+two thousand teen,2010.0,2007.0,False
+tvelve milion,12000000.0,12000000.0,True
+tventy ten,2010.0,2010.0,True
+tirty-twoo hunred,3200.0,3200.0,True
+sevn thoosands,7000.0,7000.0,True
+five,5.0,5.0,True
+ten,10.0,10.0,True
+one two three and ten,12310.0,51.0,False
+ONE MILLion three hunded and fiv,1000305.0,1000305.0,True
+"50,500 and six",50506.0,50506.0,True
+one_million_and_five,1000005.0,1000005.0,True
+2.0,2.0,2.0,True
+4.5,4.5,4.5,True
+12345.001,12345.001,12345.001,True
+7..0,7.0,7.0,True
+0.06,0.06,0.06,True
+"0,25",0.25,25.0,False
+o.45,0.45,32202.0,False
+0.1.2,0.12,32202.0,False
+0.00009,9e-05,9e-05,True
+0.01.,0.01,0.01,True
+I don't know 8,8.0,8.0,True
+"You're wrong it's not 20, it's 45",45.0,20.0,False
+I don't understand why it's 19,19.0,19.0,True

mathtext_fastapi/nlu.py

@@ -1,3 +1,4 @@
+from fuzzywuzzy import fuzz
 from mathtext_fastapi.logging import prepare_message_data_for_logging
 from mathtext.sentiment import sentiment
 from mathtext.text2int import text2int
@@ -8,27 +9,41 @@ def build_nlu_response_object(type, data, confidence):
     """ Turns nlu results into an object to send back to Turn.io
     Inputs
     - type: str - the type of nlu run (integer or sentiment-analysis)
-    - data: str - the student message
+    - data: str/int - the student message
     - confidence: - the nlu confidence score (sentiment) or '' (integer)
+
+    >>> build_nlu_response_object('integer', 8, 0)
+    {'type': 'integer', 'data': 8, 'confidence': 0}
+
+    >>> build_nlu_response_object('sentiment', 'POSITIVE', 0.99)
+    {'type': 'sentiment', 'data': 'POSITIVE', 'confidence': 0.99}
     """
     return {'type': type, 'data': data, 'confidence': confidence}
 
 
-def test_for_float_or_int(message_data, message_text):
-    nlu_response = {}
-    if type(message_text) == int or type(message_text) == float:
-        nlu_response = build_nlu_response_object('integer', message_text, '')
-        prepare_message_data_for_logging(message_data, nlu_response)
-    return nlu_response
+# def test_for_float_or_int(message_data, message_text):
+#     nlu_response = {}
+#     if type(message_text) == int or type(message_text) == float:
+#         nlu_response = build_nlu_response_object('integer', message_text, '')
+#         prepare_message_data_for_logging(message_data, nlu_response)
+#     return nlu_response
 
 
 def test_for_number_sequence(message_text_arr, message_data, message_text):
+    """ Determines if the student's message is a sequence of numbers
+
+    >>> test_for_number_sequence(['1','2','3'], {"author_id": "57787919091", "author_type": "OWNER", "contact_uuid": "df78gsdf78df", "message_body": "I am tired", "message_direction": "inbound", "message_id": "dfgha789789ag9ga", "message_inserted_at": "2023-01-10T02:37:28.487319Z", "message_updated_at": "2023-01-10T02:37:28.487319Z"}, '1, 2, 3')
+    {'type': 'integer', 'data': '1,2,3', 'confidence': 0}
+
+    >>> test_for_number_sequence(['a','b','c'], {"author_id": "57787919091", "author_type": "OWNER", "contact_uuid": "df78gsdf78df", "message_body": "I am tired", "message_direction": "inbound", "message_id": "dfgha789789ag9ga", "message_inserted_at": "2023-01-10T02:37:28.487319Z", "message_updated_at": "2023-01-10T02:37:28.487319Z"}, 'a, b, c')
+    {}
+    """
     nlu_response = {}
     if all(ele.isdigit() for ele in message_text_arr):
         nlu_response = build_nlu_response_object(
             'integer',
             ','.join(message_text_arr),
-            ''
+            0
         )
         prepare_message_data_for_logging(message_data, nlu_response)
     return nlu_response
@@ -42,6 +57,9 @@ def run_text2int_on_each_list_item(message_text_arr):
 
     Output
     - student_response_arr: list - a set of integers (32202 for error code)
+
+    >>> run_text2int_on_each_list_item(['1','2','3'])
+    [1, 2, 3]
     """
     student_response_arr = []
     for student_response in message_text_arr:
@@ -51,12 +69,63 @@ def run_text2int_on_each_list_item(message_text_arr):
 
 
 def run_sentiment_analysis(message_text):
+    """ Evaluates the sentiment of a student message
+
+    >>> run_sentiment_analysis("I am tired")
+    [{'label': 'NEGATIVE', 'score': 0.9997807145118713}]
+
+    >>> run_sentiment_analysis("I am full of joy")
+    [{'label': 'POSITIVE', 'score': 0.999882698059082}]
+    """
     # TODO: Add intent labelling here
     # TODO: Add logic to determine whether intent labeling or sentiment analysis is more appropriate (probably default to intent labeling)
     return sentiment(message_text)
 
 
+def run_intent_classification(message_text):
+    """ Process a student's message using basic fuzzy text comparison
+
+    >>> run_intent_classification("exit")
+    {'type': 'intent', 'data': 'exit', 'confidence': 1.0}
+    >>> run_intent_classification("exi")     
+    {'type': 'intent', 'data': 'exit', 'confidence': 0.86}
+    >>> run_intent_classification("eas")
+    {'type': 'intent', 'data': '', 'confidence': 0}
+    >>> run_intent_classification("hard")
+    {'type': 'intent', 'data': '', 'confidence': 0}
+    >>> run_intent_classification("hardier") 
+    {'type': 'intent', 'data': 'harder', 'confidence': 0.92}
+    """
+    label = ''
+    ratio = 0
+    nlu_response = {'type': 'intent', 'data': label, 'confidence': ratio}
+    commands = [
+        'easier',
+        'exit',
+        'harder',
+        'hint',
+        'next'
+        'stop',
+    ]
+    
+    for command in commands:
+        ratio = fuzz.ratio(command, message_text.lower())
+        if ratio > 80:
+            nlu_response['data'] = command
+            nlu_response['confidence'] = ratio / 100
+    
+    return nlu_response
+
+
 def evaluate_message_with_nlu(message_data):
+    """ Process a student's message using NLU functions and send the result
+    
+    >>> evaluate_message_with_nlu({"author_id": "57787919091", "author_type": "OWNER", "contact_uuid": "df78gsdf78df", "message_body": "8", "message_direction": "inbound", "message_id": "dfgha789789ag9ga", "message_inserted_at": "2023-01-10T02:37:28.487319Z", "message_updated_at": "2023-01-10T02:37:28.487319Z"})
+    {'type': 'integer', 'data': 8, 'confidence': 0}
+
+    >>> evaluate_message_with_nlu({"author_id": "57787919091", "author_type": "OWNER", "contact_uuid": "df78gsdf78df", "message_body": "I am tired", "message_direction": "inbound", "message_id": "dfgha789789ag9ga", "message_inserted_at": "2023-01-10T02:37:28.487319Z", "message_updated_at": "2023-01-10T02:37:28.487319Z"})
+    {'type': 'sentiment', 'data': 'NEGATIVE', 'confidence': 0.9997807145118713}
+    """
     # Keeps system working with two different inputs - full and filtered @event object
     try:
         message_text = message_data['message_body']
@@ -76,6 +145,13 @@ def evaluate_message_with_nlu(message_data):
     number_api_resp = text2int(message_text.lower())
 
     if number_api_resp == 32202:
+        print("MESSAGE TEXT")
+        print(message_text)
+        print("============")
+        intent_api_response = run_intent_classification(message_text)
+        if intent_api_response['data']:
+            return intent_api_response
+
         sentiment_api_resp = sentiment(message_text)
         nlu_response = build_nlu_response_object(
             'sentiment',

requirements.txt

@@ -1,9 +1,11 @@
 dill
-en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.4.1/en_core_web_sm-3.4.1-py3-none-any.whl  
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.4.1/en_core_web_sm-3.4.1-py3-none-any.whl
+fuzzywuzzy
 jsonpickle
 mathtext @ git+https://gitlab.com/tangibleai/community/mathtext@main
 fastapi==0.74.*
 pydantic==1.10.*
+python-Levenshtein
 requests==2.27.*
 sentencepiece==0.1.*
 supabase

scripts/make_request.py

@@ -58,8 +58,8 @@ def run_simulated_request(endpoint, sample_answer, context=None):
     print(request)
 
 
-# run_simulated_request('sentiment-analysis', 'I reject it')
-# run_simulated_request('text2int', 'seven thousand nine hundred fifty seven')
+run_simulated_request('sentiment-analysis', 'I reject it')
+run_simulated_request('text2int', 'seven thousand nine hundred fifty seven')
 run_simulated_request('nlu', 'test message')
 run_simulated_request('nlu', 'eight')
 run_simulated_request('nlu', 'is it 8')