conversation clause tests + categorization data

data/model/sent_func/fingerprint.pb

@@ -1 +1 @@
-ｼⅶ喜矣ﾊﾔ
ﾘﾁ<EFBE98><EFBE81>ﾌ鈺
･ﾗ･帝<EFBDA5>G 鼓ｳ愠筺<E684A0>
(<28>ﾜｪﾌﾞﾈ<EFBE9E>
2

+╧╗╟├З▓╛Ы╛
 ╠М╠▐б▀╥╥
╔в╔▓ИКЙG сМ╘И┐Ь▓б├
(║ён╓┘Ъ╜йф
2

data/semantic/sent_func/test.csv

@@ -10,4 +10,15 @@
 0,1,0,0,"Your hat is Blue"
 0,1,0,0,"Your hat is blue"
 0,1,0,0,"My name is John"
-0,1,0,0,"Your name is John"
+0,1,0,0,"Your name is John"
+0,0,1,0,"What time is the meeting tomorrow?"
+0,0,0,1,"Tell me a joke!"
+0,0,1,0,"Why did you suggest using an bucket to shovel dirt?"
+0,0,1,0,"It's too cold in here; can you turn up the heat?"
+0,0,1,0,"Who are you?"
+0,0,1,0,"Where are you from?"
+0,0,1,0,"What is your name?"
+0,0,1,0,"Who is responsible for maintaining the project documentation?"
+0,0,1,0,"What is the capital of France?"
+0,0,0,1,"Explain the difference between god and man."
+0,0,1,0,"Where did your staff go?"

data/semantic/sent_func/train.csv

@@ -10,4 +10,15 @@
 0,1,0,0,"Your hat is Blue"
 0,1,0,0,"Your hat is blue"
 0,1,0,0,"My name is John"
-0,1,0,0,"Your name is John"
+0,1,0,0,"Your name is John"
+0,0,1,0,"What time is the meeting tomorrow?"
+0,0,0,1,"Tell me a joke!"
+0,0,1,0,"Why did you suggest using an bucket to shovel dirt?"
+0,0,1,0,"It's too cold in here; can you turn up the heat?"
+0,0,1,0,"Who are you?"
+0,0,1,0,"Where are you from?"
+0,0,1,0,"What is your name?"
+0,0,1,0,"Who is responsible for maintaining the project documentation?"
+0,0,1,0,"What is the capital of France?"
+0,0,0,1,"Explain the difference between god and man."
+0,0,1,0,"Where did your staff go?"

src/main/java/org/studiorailgun/conversation/parser/depend/Clause.java

@@ -122,13 +122,36 @@ public class Clause {
                     rVal.coordinator = new Coordinator(child);
                 } break;
 
-                //A subject in a compound sentence
+                //A subject of a dependent clause in a compound sentence
                 case "compound modifier":{
                     Argument arg = new Argument(child);
                     rVal.arguments.add(arg);
                     rVal.subject = arg;
                 } break;
 
+                //An argument of a verb or adjective
+                case "xclausal complement": {
+                    Argument arg = new Argument(child);
+                    rVal.arguments.add(arg);
+                } break;
+
+                //an auxiliary verb for the predicate
+                case "auxiliary": {
+                    pred.setAuxiliary(child);
+                } break;
+
+                //An adverbial clause modifier
+                case "advcl_preposition": {
+                    Adjunct adj = new Adjunct(child);
+                    rVal.adjuncts.add(adj);
+                } break;
+
+                //Something that is combined with the verb to create a new meaning
+                //eg "turn" -> "turn up"
+                case "phrasal verb particle": {
+                    pred.setPhrasalVerbParticle(child);
+                } break;
+
 
                 //
                 //clauses
@@ -146,6 +169,30 @@ public class Clause {
                     rVal.arguments.add(arg);
                 } break;
 
+                /**
+                 * A dependent clause
+                 */
+                case "dependent": {
+                    Clause clause = Clause.parse(graph, child);
+                    rVal.clauses.add(clause);
+                } break;
+
+                /**
+                 * A paratactic construct (A clause that is very loosely related to the main clause)
+                 */
+                case "parataxis": {
+                    Clause clause = Clause.parse(graph, child);
+                    rVal.clauses.add(clause);
+                } break;
+
+
+
+
+                //
+                // Misc
+                //
+
+
                 //cases to ignore
                 case "punctuation": {
                     continue;

src/main/java/org/studiorailgun/conversation/parser/depend/Predicate.java

@@ -26,6 +26,16 @@ public class Predicate {
      */
     boolean existential;
 
+    /**
+     * The auxiliary verb for this predicate
+     */
+    IndexedWord auxiliary;
+
+    /**
+     * The phrasal verb particle for this predicate
+     */
+    IndexedWord phrasalVerbParticle;
+
     /**
      * Constructor
      * @param root The root of the predicate
@@ -74,4 +84,20 @@ public class Predicate {
         return existential;
     }
 
+    /**
+     * Sets the auxiliary verb for this predicate
+     * @param auxiliary The auxiliary verb
+     */
+    public void setAuxiliary(IndexedWord auxiliary){
+        this.auxiliary = auxiliary;
+    }
+
+    /**
+     * Sets the phrasal verb particle for the predicate
+     * @param phrasalVerbParticle the phrasal very particle
+     */
+    public void setPhrasalVerbParticle(IndexedWord phrasalVerbParticle){
+        this.phrasalVerbParticle = phrasalVerbParticle;
+    }
+
 }

src/main/python/conversation/sentence/function.py

@@ -1,7 +1,7 @@
 import tensorflow as tf
 keras = tf.keras
 from tensorflow import Tensor
-from keras.api.layers import TextVectorization, Embedding, LSTM, Dense, Input
+from keras.api.layers import TextVectorization, Embedding, LSTM, Dense, Bidirectional
 from keras.api.models import Sequential
 import numpy as np
 import numpy.typing as npt
@@ -124,7 +124,8 @@ model: Sequential = Sequential([
     keras.Input(shape=(1,), dtype=tf.string),
     textVec,
     Embedding(max_features + 1, embedding_dim),
-    LSTM(128),
+    Bidirectional(LSTM(256)),
+    Dense(64, activation='relu'),
     Dense(num_classes, activation='sigmoid')
 ])
 

src/test/java/org/studiorailgun/ConversationClauseTests.java

@@ -0,0 +1,206 @@
+package org.studiorailgun;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import org.junit.Test;
+import org.studiorailgun.conversation.parser.NLPParser;
+import org.studiorailgun.conversation.parser.depend.Clause;
+
+import edu.stanford.nlp.pipeline.CoreDocument;
+import edu.stanford.nlp.pipeline.CoreSentence;
+import edu.stanford.nlp.semgraph.SemanticGraph;
+
+/**
+ * Test parsing clauses that we might see in a conversation
+ */
+public class ConversationClauseTests {
+    
+    @Test
+    public void testParseConversationClause1(){
+        NLPParser.init();
+        CoreDocument doc = NLPParser.parse("What time is the meeting tomorrow?");
+        CoreSentence sentence = doc.sentences().get(0);
+        SemanticGraph graph = sentence.dependencyParse();
+        Clause struct = Clause.parse(graph);
+
+        //test number of returns
+        assertNotNull(struct.getPredicate());
+        assertEquals(struct.getArguments().size(),2);
+
+        //test returned data
+        assertEquals(struct.getPredicate().getRoot().originalText(), "is"); //should be copular verb
+        assertEquals(struct.getArguments().get(0).getRoot().originalText(), "time"); //should be the subject
+    }
+
+    @Test
+    public void testParseConversationClause2(){
+        NLPParser.init();
+        CoreDocument doc = NLPParser.parse("Tell me a joke!");
+        CoreSentence sentence = doc.sentences().get(0);
+        SemanticGraph graph = sentence.dependencyParse();
+        Clause struct = Clause.parse(graph);
+
+        //test number of returns
+        assertNotNull(struct.getPredicate());
+        assertEquals(struct.getArguments().size(),2);
+
+        //test returned data
+        assertEquals(struct.getPredicate().getRoot().originalText(), "Tell"); //should be copular verb
+        assertEquals(struct.getArguments().get(0).getRoot().originalText(), "me"); //should be the subject
+    }
+
+    @Test
+    public void testParseConversationClause3(){
+        NLPParser.init();
+        CoreDocument doc = NLPParser.parse("Why did you suggest using an bucket to shovel dirt?");
+        CoreSentence sentence = doc.sentences().get(0);
+        SemanticGraph graph = sentence.dependencyParse();
+        Clause struct = Clause.parse(graph);
+
+        //test number of returns
+        assertNotNull(struct.getPredicate());
+        assertEquals(struct.getArguments().size(),1);
+
+        //test returned data
+        assertEquals(struct.getPredicate().getRoot().originalText(), "suggest"); //should be copular verb
+        assertEquals(struct.getArguments().get(0).getRoot().originalText(), "you"); //should be the subject
+    }
+
+    @Test
+    public void testParseConversationClause4(){
+        NLPParser.init();
+        CoreDocument doc = NLPParser.parse("It's too cold in here; can you turn up the heat?");
+        CoreSentence sentence = doc.sentences().get(0);
+        SemanticGraph graph = sentence.dependencyParse();
+        Clause struct = Clause.parse(graph);
+
+        //test number of returns
+        assertNotNull(struct.getPredicate());
+        assertEquals(struct.getArguments().size(),1);
+
+        //test returned data
+        assertEquals(struct.getPredicate().getRoot().originalText(), "cold"); //should be copular verb
+        assertEquals(struct.getArguments().get(0).getRoot().originalText(), "It"); //should be the subject
+    }
+
+    @Test
+    public void testParseConversationClause5(){
+        NLPParser.init();
+        CoreDocument doc = NLPParser.parse("Who are you?");
+        CoreSentence sentence = doc.sentences().get(0);
+        SemanticGraph graph = sentence.dependencyParse();
+        Clause struct = Clause.parse(graph);
+
+        //test number of returns
+        assertNotNull(struct.getPredicate());
+        assertEquals(struct.getArguments().size(),1);
+
+        //test returned data
+        assertEquals(struct.getPredicate().getRoot().originalText(), "Who"); //should be copular verb
+        assertEquals(struct.getArguments().get(0).getRoot().originalText(), "you"); //should be the subject
+    }
+
+    @Test
+    public void testParseConversationClause6(){
+        NLPParser.init();
+        CoreDocument doc = NLPParser.parse("Where are you from?");
+        CoreSentence sentence = doc.sentences().get(0);
+        SemanticGraph graph = sentence.dependencyParse();
+        Clause struct = Clause.parse(graph);
+
+        //test number of returns
+        assertNotNull(struct.getPredicate());
+        assertEquals(struct.getArguments().size(),1);
+
+        //test returned data
+        assertEquals(struct.getPredicate().getRoot().originalText(), "are"); //should be copular verb
+        assertEquals(struct.getArguments().get(0).getRoot().originalText(), "you"); //should be the subject
+    }
+
+    @Test
+    public void testParseConversationClause7(){
+        NLPParser.init();
+        CoreDocument doc = NLPParser.parse("What is your name?");
+        CoreSentence sentence = doc.sentences().get(0);
+        SemanticGraph graph = sentence.dependencyParse();
+        Clause struct = Clause.parse(graph);
+
+        //test number of returns
+        assertNotNull(struct.getPredicate());
+        assertEquals(struct.getArguments().size(),1);
+
+        //test returned data
+        assertEquals(struct.getPredicate().getRoot().originalText(), "What"); //should be copular verb
+        assertEquals(struct.getArguments().get(0).getRoot().originalText(), "name"); //should be the subject
+    }
+
+    @Test
+    public void testParseConversationClause8(){
+        NLPParser.init();
+        CoreDocument doc = NLPParser.parse("Who is responsible for maintaining the project documentation?");
+        CoreSentence sentence = doc.sentences().get(0);
+        SemanticGraph graph = sentence.dependencyParse();
+        Clause struct = Clause.parse(graph);
+
+        //test number of returns
+        assertNotNull(struct.getPredicate());
+        assertEquals(struct.getArguments().size(),1);
+
+        //test returned data
+        assertEquals(struct.getPredicate().getRoot().originalText(), "responsible"); //should be copular verb
+        assertEquals(struct.getArguments().get(0).getRoot().originalText(), "Who"); //should be the subject
+    }
+
+    @Test
+    public void testParseConversationClause9(){
+        NLPParser.init();
+        CoreDocument doc = NLPParser.parse("What is the capital of France?");
+        CoreSentence sentence = doc.sentences().get(0);
+        SemanticGraph graph = sentence.dependencyParse();
+        Clause struct = Clause.parse(graph);
+
+        //test number of returns
+        assertNotNull(struct.getPredicate());
+        assertEquals(struct.getArguments().size(),1);
+
+        //test returned data
+        assertEquals(struct.getPredicate().getRoot().originalText(), "What"); //should be copular verb
+        assertEquals(struct.getArguments().get(0).getRoot().originalText(), "capital"); //should be the subject
+    }
+
+    @Test
+    public void testParseConversationClause10(){
+        NLPParser.init();
+        CoreDocument doc = NLPParser.parse("Explain the difference between god and man.");
+        CoreSentence sentence = doc.sentences().get(0);
+        SemanticGraph graph = sentence.dependencyParse();
+        Clause struct = Clause.parse(graph);
+
+        //test number of returns
+        assertNotNull(struct.getPredicate());
+        assertEquals(struct.getArguments().size(),1);
+
+        //test returned data
+        assertEquals(struct.getPredicate().getRoot().originalText(), "Explain"); //should be copular verb
+        assertEquals(struct.getArguments().get(0).getRoot().originalText(), "difference"); //should be the subject
+    }
+
+    @Test
+    public void testParseConversationClause11(){
+        NLPParser.init();
+        CoreDocument doc = NLPParser.parse("Where did your staff go?");
+        CoreSentence sentence = doc.sentences().get(0);
+        SemanticGraph graph = sentence.dependencyParse();
+        Clause struct = Clause.parse(graph);
+
+        //test number of returns
+        assertNotNull(struct.getPredicate());
+        assertEquals(struct.getArguments().size(),1);
+
+        //test returned data
+        assertEquals(struct.getPredicate().getRoot().originalText(), "go"); //should be copular verb
+        assertEquals(struct.getArguments().get(0).getRoot().originalText(), "staff"); //should be the subject
+    }
+
+}

src/test/java/org/studiorailgun/SimpleClauseTests.java

@@ -172,4 +172,21 @@ public class SimpleClauseTests {
         assertEquals(struct.getArguments().get(0).getRoot().originalText(), "ball"); //should be the subject
     }
 
+    @Test
+    public void testMacroStructureParse10(){
+        NLPParser.init();
+        CoreDocument doc = NLPParser.parse("She appears intelligent.");
+        CoreSentence sentence = doc.sentences().get(0);
+        SemanticGraph graph = sentence.dependencyParse();
+        Clause struct = Clause.parse(graph);
+
+        //test number of returns
+        assertNotNull(struct.getPredicate());
+        assertEquals(struct.getArguments().size(),2);
+
+        //test returned data
+        assertEquals(struct.getPredicate().getRoot().originalText(), "appears"); //should be copular verb
+        assertEquals(struct.getArguments().get(0).getRoot().originalText(), "She"); //should be the subject
+    }
+
 }