Commit 2f8dba52 authored by Raul Sirel's avatar Raul Sirel
Browse files

Merge branch 'remove_empty_lemmas_value' into 'master'

Hide lemmas value when it's None, remove None value when evaluating for duplicates.

See merge request !11
parents 99e2e636 ba61335e
Pipeline #5590 passed with stage
in 14 minutes and 54 seconds
......@@ -98,9 +98,20 @@ class Document:
self.add_fact(fact)
@staticmethod
def handle_null_values_in_facts(facts: List[dict]):
container = []
if facts:
for fact in facts:
new_fact = {key: value for key, value in fact.items() if value is not None}
container.append(new_fact)
return container
@staticmethod
def remove_duplicate_facts(facts: List[dict]):
if facts:
facts = Document.handle_null_values_in_facts(facts)
set_of_jsons = {json.dumps(fact, sort_keys=True, ensure_ascii=False) for fact in facts}
without_duplicates = [json.loads(unique_fact) for unique_fact in set_of_jsons]
return without_duplicates
......
......@@ -26,10 +26,12 @@ class Fact:
def to_json(self):
return {
container = {
"str_val": self.fact_value,
"spans": json.dumps(self.spans),
"fact": self.fact_type,
"doc_path": self.doc_path,
"lemma": self.fact_lemma
}
if self.fact_lemma:
container["lemma"] = self.fact_lemma
return container
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment