Commit 35497c3c authored by Wael Ramadan's avatar Wael Ramadan
Browse files

refactor no need to sets of json

parent 28f42f75
Pipeline #8006 passed with stage
in 8 minutes and 25 seconds
import json
import math
from typing import List, Optional
......@@ -144,10 +143,8 @@ class Document:
def remove_duplicate_facts(facts: List[dict]):
if facts:
facts = Document.handle_null_values_in_facts(facts)
set_of_jsons = {json.dumps(fact, sort_keys=True, ensure_ascii=False) for fact in facts}
without_duplicates = [json.loads(unique_fact) for unique_fact in set_of_jsons]
without_duplicates_ignored_keys = list(Document.remove_duplicates_with_ignored_keys(without_duplicates, ["id", "source"]))
return without_duplicates_ignored_keys
without_duplicates = list(Document.remove_duplicates_with_ignored_keys(facts, ["id", "source"]))
return without_duplicates
else:
return []
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment