Commit ba61335e authored by Marko Kollo's avatar Marko Kollo 😄
Browse files

Hide lemmas value when it's None, remove None value when evaluating for duplicates.

parent 99e2e636
Pipeline #5584 passed with stage
in 29 minutes and 23 seconds
......@@ -98,9 +98,20 @@ class Document:
def handle_null_values_in_facts(facts: List[dict]):
container = []
if facts:
for fact in facts:
new_fact = {key: value for key, value in fact.items() if value is not None}
return container
def remove_duplicate_facts(facts: List[dict]):
if facts:
facts = Document.handle_null_values_in_facts(facts)
set_of_jsons = {json.dumps(fact, sort_keys=True, ensure_ascii=False) for fact in facts}
without_duplicates = [json.loads(unique_fact) for unique_fact in set_of_jsons]
return without_duplicates
......@@ -26,10 +26,12 @@ class Fact:
def to_json(self):
return {
container = {
"str_val": self.fact_value,
"spans": json.dumps(self.spans),
"fact": self.fact_type,
"doc_path": self.doc_path,
"lemma": self.fact_lemma
if self.fact_lemma:
container["lemma"] = self.fact_lemma
return container
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment