Hide lemmas value when it's None, remove None value when evaluating for duplicates.

texta_mlp/document.py

@@ -98,9 +98,20 @@ class Document:
                 self.add_fact(fact)
 
 
+    @staticmethod
+    def handle_null_values_in_facts(facts: List[dict]):
+        container = []
+        if facts:
+            for fact in facts:
+                new_fact = {key: value for key, value in fact.items() if value is not None}
+                container.append(new_fact)
+        return container
+
+
     @staticmethod
     def remove_duplicate_facts(facts: List[dict]):
         if facts:
+            facts = Document.handle_null_values_in_facts(facts)
             set_of_jsons = {json.dumps(fact, sort_keys=True, ensure_ascii=False) for fact in facts}
             without_duplicates = [json.loads(unique_fact) for unique_fact in set_of_jsons]
             return without_duplicates

texta_mlp/fact.py

@@ -26,10 +26,12 @@ class Fact:
 
 
     def to_json(self):
-        return {
+        container = {
             "str_val": self.fact_value,
             "spans": json.dumps(self.spans),
             "fact": self.fact_type,
             "doc_path": self.doc_path,
-            "lemma": self.fact_lemma
         }
+        if self.fact_lemma:
+            container["lemma"] = self.fact_lemma
+        return container