Commit 239e172b authored by Raul Sirel's avatar Raul Sirel
Browse files

update requirements & taskman

parent 7742ca79
Pipeline #4486 failed with stages
in 7 minutes and 33 seconds
......@@ -8,5 +8,3 @@ lang-trans
regex
pytest
phonenumberslite
redis==3.*
celery==4.*
\ No newline at end of file
......@@ -3,6 +3,7 @@ from celery import Celery
from typing import Optional
import logging
from texta_mlp.mlp_docparser import MLPDocParser
from texta_mlp.mlp import MLP
from settings import (
......@@ -14,10 +15,6 @@ from settings import (
)
import logging
from texta_mlp.mlp_processor import MLPProcessor
# Create Celery app with proper conf
app = Celery("worker")
app.conf.broker_url = MLP_WORKER_BROKER
......@@ -44,14 +41,17 @@ def load_mlp():
logging.info("Successfully loaded MLP models.")
#@shared_task
#def mlp(document: list, field: list, analyzers: list):
# try:
# load_mlp()
# processed = ml_processor.process_docs(docs=document, doc_paths=field, analyzers=analyzers)
# return processed
# except Exception as e:
# return e
@shared_task
def mlp(text: str):
"""
Analyzes a string.
"""
try:
load_mlp()
processed = ml_processor.process(text)
return processed
except Exception as e:
return e
@shared_task
......@@ -65,9 +65,9 @@ def mlp_docparser(task_results: dict):
files_to_remove = task_results.get("files_to_remove", [])
file_hash = task_results.get("file_hash", "")
# mlp processor
mlp_analyzer = MLPProcessor(ml_processor)
mlp_analyzer = MLPDocParser(ml_processor)
# apply MLP
document_generator = mlp_analyzer.apply_mlp(document_list, analyzers=["lemmas", "ner", "transliteration", "pos_tags", "entities", "addresses", "emails", "phone_strict"])
document_generator = mlp_analyzer.apply_mlp(document_list)
document_list = list(document_generator)
logging.getLogger(INFO_LOGGER).info("Sucessfully processed with MLP using mlp_docparser task.")
return {"document_list": document_list, "meta": {"files_to_remove": files_to_remove, "hash": file_hash}}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment