geografia backend: ccq demo
agropunx
2 years ago
parent
bb97cacc97
commit
d32f66b07f
@ -0,0 +1,288 @@
|
|||||||
|
#!/home/agropunx/anaconda3/envs/geografia/bin/python
|
||||||
|
|
||||||
|
import time, subprocess, copy,os, sys, numpy as np, pandas as pd
|
||||||
|
import config
|
||||||
|
import spacy
|
||||||
|
|
||||||
|
def spoken_kind(doc):
|
||||||
|
interrogative = '?' in doc.text.strip()[-4:]
|
||||||
|
if interrogative:
|
||||||
|
sub_kind = [ 'closed','open'][np.sum([tok.text in config.open_question_words for tok in doc]) >= 1]
|
||||||
|
out = f"interrogative_{sub_kind}"
|
||||||
|
else:
|
||||||
|
out = ['affirmative', 'negative'][np.sum([tok.text in config.negative_words for tok in doc]) >= 1]
|
||||||
|
return out
|
||||||
|
|
||||||
|
def spoken_subject(doc):
|
||||||
|
for token in doc:
|
||||||
|
if "subj" in token.dep_:
|
||||||
|
subtree = list(token.subtree)
|
||||||
|
start = subtree[0].i
|
||||||
|
end = subtree[-1].i + 1
|
||||||
|
return doc[start:end]
|
||||||
|
|
||||||
|
def spoken_dobject(doc):
|
||||||
|
for token in doc:
|
||||||
|
if "dobj" in token.dep_:
|
||||||
|
subtree = list(token.subtree)
|
||||||
|
start = subtree[0].i
|
||||||
|
end = subtree[-1].i + 1
|
||||||
|
return doc[start:end]
|
||||||
|
|
||||||
|
def spoken_verb(doc):
|
||||||
|
verb_related=[]
|
||||||
|
for i,token in enumerate(doc):
|
||||||
|
if token.dep_ in ['ROOT','cop', 'advmod','aux']:
|
||||||
|
if token.text not in config.time_terms+config.negative_words:
|
||||||
|
verb_related.append(i)
|
||||||
|
return [doc[i] for i in verb_related]
|
||||||
|
|
||||||
|
def spoken_time(doc):
|
||||||
|
for ent in doc.ents:
|
||||||
|
if ent.label_ in ['DATE', 'TIME']:
|
||||||
|
return doc[ent.start:ent.end]
|
||||||
|
for i,tok in enumerate(doc):
|
||||||
|
if tok.text in config.time_terms:
|
||||||
|
return doc[i]
|
||||||
|
|
||||||
|
def spoken_negative(doc):
|
||||||
|
for i,tok in enumerate(doc):
|
||||||
|
if tok.text in config.negative_words:
|
||||||
|
return i
|
||||||
|
|
||||||
|
|
||||||
|
spacy.tokens.Doc.set_extension("kind", getter=lambda doc: spoken_kind(doc), force=True)
|
||||||
|
spacy.tokens.Doc.set_extension("subject", getter=lambda doc: spoken_subject(doc), force=True)
|
||||||
|
spacy.tokens.Doc.set_extension("dobject", getter=lambda doc: spoken_dobject(doc), force=True)
|
||||||
|
spacy.tokens.Doc.set_extension("verb", getter=lambda doc: spoken_verb(doc), force=True)
|
||||||
|
spacy.tokens.Doc.set_extension("time", getter=lambda doc: spoken_time(doc), force=True)
|
||||||
|
spacy.tokens.Doc.set_extension("negative", getter=lambda doc: spoken_negative(doc), force=True)
|
||||||
|
|
||||||
|
def bash_command(cmd):
|
||||||
|
subprocess.Popen(cmd, shell=True, executable='/bin/bash')
|
||||||
|
|
||||||
|
class CCQ:
|
||||||
|
|
||||||
|
def __init__(self, model_name ="it_core_news_lg", ner_model_name="it_nerIta_trf"):
|
||||||
|
if ner_model_name:
|
||||||
|
self.nlp = spacy.load(model_name, exclude=['ner'])
|
||||||
|
self.ner = spacy.load(ner_model_name)
|
||||||
|
else:
|
||||||
|
self.nlp = spacy.load(model_name)
|
||||||
|
self.ner = None
|
||||||
|
|
||||||
|
def analyze(self,sentence,mode='spoken'):
|
||||||
|
if mode=='spoken':
|
||||||
|
|
||||||
|
doc = self.nlp(sentence)
|
||||||
|
if self.ner:
|
||||||
|
nerdoc= self.ner(doc)
|
||||||
|
doc.ents = nerdoc.ents
|
||||||
|
doc._.time = nerdoc._.time
|
||||||
|
elif mode=='gloss':
|
||||||
|
doc=sentence
|
||||||
|
else:
|
||||||
|
raise Exception(f'unknown mode {mode}')
|
||||||
|
return doc
|
||||||
|
|
||||||
|
def check_feasibility(self, doc,valid_tokens, mode='spoken', rules= config.rules):
|
||||||
|
rules = rules[mode]
|
||||||
|
if mode=='spoken':
|
||||||
|
if isinstance(doc, str):
|
||||||
|
doc = self.nlp(doc)
|
||||||
|
status = {
|
||||||
|
'max_tokens': len(doc)<rules['max_tokens'],
|
||||||
|
'valid_tokens' : sum(valid_tokens)<rules['valid_tokens'],
|
||||||
|
'punct':sum([t.is_punct for t in doc])<=1
|
||||||
|
}
|
||||||
|
if doc._.negative:
|
||||||
|
status['negative_interrogative'] = 'interrogative' not in doc._.kind
|
||||||
|
if doc._.verb:
|
||||||
|
status['verb'] =len(doc._.verb)<rules['max_verb_terms']
|
||||||
|
if doc._.time:
|
||||||
|
status['time'] = len(doc._.time.text.split())<rules['max_time_terms']
|
||||||
|
if doc._.subject:
|
||||||
|
status['subject'] = len(doc._.subject.text.split())<rules['max_subject_terms']
|
||||||
|
elif mode=='gloss':
|
||||||
|
status = {
|
||||||
|
'subject' : sum([t[1]=='subject' for t in doc]) < rules['subject'],
|
||||||
|
'verb': sum([t[1]=='verb' for t in doc]) < rules['verb'],
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
raise Exception(f'unknown mode {mode}')
|
||||||
|
|
||||||
|
status['glob'] = sum(list(status.values())) == len(status)
|
||||||
|
|
||||||
|
return status
|
||||||
|
|
||||||
|
def check_kind(self, doc,mode='spoken'):
|
||||||
|
if mode=='spoken':
|
||||||
|
if isinstance(doc,str):
|
||||||
|
doc=self.nlp(doc)
|
||||||
|
out = doc._.kind
|
||||||
|
elif mode=='gloss':
|
||||||
|
if sum([True for t in doc if '?' in t[0]])>=1:
|
||||||
|
if sum([True for t in doc if t[0] in config.open_question_words])>=1:
|
||||||
|
out = 'interrogative_open'
|
||||||
|
else:
|
||||||
|
out = 'interrogative_closed'
|
||||||
|
else:
|
||||||
|
if sum([True for t in doc if t[0] in config.negative_words]) >= 1:
|
||||||
|
out = 'negative'
|
||||||
|
else:
|
||||||
|
out = 'affirmative'
|
||||||
|
else:
|
||||||
|
raise Exception(f'unknown mode {mode}')
|
||||||
|
return out
|
||||||
|
|
||||||
|
def tok_validator(self, doc, mode='spoken'):
|
||||||
|
if mode=='spoken':
|
||||||
|
if isinstance(doc,str):
|
||||||
|
doc=self.nlp(doc)
|
||||||
|
valid_tokens = [not((tok.pos_ in ['DET','ADP']) or (tok.text in config.spoken_stopwords) or (tok.is_punct) or (tok.text=='?')) for tok in doc]#'AUX'
|
||||||
|
elif mode=='gloss':
|
||||||
|
valid_tokens = [True for _ in doc]
|
||||||
|
else:
|
||||||
|
raise Exception(f'unknown mode {mode}')
|
||||||
|
return valid_tokens
|
||||||
|
|
||||||
|
def reorder(self,doc, valid_tokens=[], direction='gloss2spoken', kind='affirmative'):
|
||||||
|
reordered = []
|
||||||
|
tail_idx = []
|
||||||
|
idx_done = np.array([False for _ in range(len(doc))])
|
||||||
|
|
||||||
|
if direction=='spoken2gloss':
|
||||||
|
# SOV -> SVO
|
||||||
|
assert isinstance(doc,spacy.tokens.doc.Doc), type(doc)
|
||||||
|
assert doc._.kind in ['affirmative', 'negative', 'interrogative_open', 'interrogative_closed'], doc._.kind
|
||||||
|
|
||||||
|
if len(valid_tokens)==0:
|
||||||
|
valid_tokens = [True for _ in doc]
|
||||||
|
|
||||||
|
if doc._.time:
|
||||||
|
if isinstance(doc._.time,spacy.tokens.span.Span):
|
||||||
|
tidx = [t for t in range(doc._.time.start, doc._.time.end)]
|
||||||
|
idx_done[tidx] = True
|
||||||
|
else:
|
||||||
|
tidx = [i for i,t in enumerate(doc) if t.idx==doc._.time.idx]
|
||||||
|
idx_done[tidx[0]] = True
|
||||||
|
reordered += [t for t in tidx if valid_tokens[t]]
|
||||||
|
|
||||||
|
if 'interrogative' in kind:
|
||||||
|
if 'open' in kind:
|
||||||
|
open_question_term_idx = [i for i,tok in enumerate(doc) if tok.text in config.open_question_words][0]
|
||||||
|
if doc._.subject:
|
||||||
|
if isinstance(doc._.subject, spacy.tokens.span.Span):
|
||||||
|
tidx = [t for t in range(doc._.subject.start, doc._.subject.end) if t!=open_question_term_idx]
|
||||||
|
idx_done[tidx] = True
|
||||||
|
else:
|
||||||
|
tidx = [i for i, t in enumerate(doc) if t.idx == doc._.subject.idx and i!=open_question_term_idx]
|
||||||
|
idx_done[tidx[0]] = True
|
||||||
|
reordered += [t for t in tidx if valid_tokens[t]]
|
||||||
|
if doc._.verb:
|
||||||
|
substart = [t.idx for t in doc._.verb]
|
||||||
|
tidx = [i for i, tok in enumerate(doc) if tok.idx in substart and i!=open_question_term_idx]
|
||||||
|
reordered += [t for t in tidx if valid_tokens[t]]
|
||||||
|
idx_done[tidx] = True
|
||||||
|
else:
|
||||||
|
if doc._.subject:
|
||||||
|
if isinstance(doc._.subject, spacy.tokens.span.Span):
|
||||||
|
tidx = [t for t in range(doc._.subject.start, doc._.subject.end)]
|
||||||
|
idx_done[tidx] = True
|
||||||
|
else:
|
||||||
|
tidx = [i for i, t in enumerate(doc) if t.idx == doc._.subject.idx]
|
||||||
|
idx_done[tidx[0]] = True
|
||||||
|
tail_idx += [t for t in tidx if valid_tokens[t]]
|
||||||
|
|
||||||
|
if doc._.verb:
|
||||||
|
substart = [t.idx for t in doc._.verb]
|
||||||
|
tidx = [i for i, tok in enumerate(doc) if tok.idx in substart]
|
||||||
|
tail_idx = [t for t in tidx if valid_tokens[t]]
|
||||||
|
idx_done[tidx] = True
|
||||||
|
else:
|
||||||
|
if doc._.subject:
|
||||||
|
if isinstance(doc._.subject, spacy.tokens.span.Span):
|
||||||
|
tidx = [t for t in range(doc._.subject.start, doc._.subject.end)]
|
||||||
|
idx_done[tidx] = True
|
||||||
|
else:
|
||||||
|
tidx = [i for i, t in enumerate(doc) if t.idx == doc._.subject.idx]
|
||||||
|
idx_done[tidx[0]] = True
|
||||||
|
|
||||||
|
reordered += [t for t in tidx if valid_tokens[t]]
|
||||||
|
|
||||||
|
if doc._.verb:
|
||||||
|
substart = [t.idx for t in doc._.verb]
|
||||||
|
tidx = [i for i, tok in enumerate(doc) if tok.idx in substart]
|
||||||
|
tail_idx += [t for t in tidx if valid_tokens[t]]
|
||||||
|
idx_done[tidx] = True
|
||||||
|
|
||||||
|
if doc._.negative:
|
||||||
|
tidx = doc._.negative
|
||||||
|
tail_idx += [tidx]
|
||||||
|
idx_done[tidx] = True
|
||||||
|
|
||||||
|
for i, done in enumerate(idx_done):
|
||||||
|
if not done:
|
||||||
|
if valid_tokens[i]:
|
||||||
|
reordered.append(i)
|
||||||
|
idx_done[i] = True
|
||||||
|
reordered += tail_idx
|
||||||
|
assert sum(idx_done)==len(doc)
|
||||||
|
reordered = [doc[r].lemma_.lower() for r in reordered]
|
||||||
|
if doc._.negative:
|
||||||
|
reordered[-1]='no'
|
||||||
|
else:
|
||||||
|
if 'interrogative' in kind:
|
||||||
|
if 'open' in kind:
|
||||||
|
wh = doc[open_question_term_idx].text.lower()
|
||||||
|
if wh!=reordered[-1]:
|
||||||
|
reordered.append(wh)
|
||||||
|
reordered.append('?')
|
||||||
|
|
||||||
|
elif direction=='gloss2spoken':
|
||||||
|
# SVO -> SOV
|
||||||
|
for i,t in enumerate(doc):
|
||||||
|
if t[1]=='subject':
|
||||||
|
reordered.append(t[0])
|
||||||
|
idx_done[i]=True
|
||||||
|
|
||||||
|
for i,t in enumerate(doc):
|
||||||
|
if t[0] in config.open_question_words:
|
||||||
|
reordered.append(t[0])
|
||||||
|
idx_done[i]=True
|
||||||
|
|
||||||
|
for i,t in enumerate(doc):
|
||||||
|
if t[1]=='verb':
|
||||||
|
reordered.append(t[0])
|
||||||
|
idx_done[i]=True
|
||||||
|
|
||||||
|
for i,t in enumerate(doc):
|
||||||
|
if not idx_done[i]:
|
||||||
|
reordered.append(t[0])
|
||||||
|
idx_done[i]=True
|
||||||
|
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise Exception(f'unknown direction {direction}')
|
||||||
|
assert len(doc)==sum(idx_done)
|
||||||
|
return reordered
|
||||||
|
|
||||||
|
def translate(self,sentence, direction='spoken2gloss'):
|
||||||
|
assert direction in ['spoken2gloss','gloss2spoken'], direction
|
||||||
|
mode = direction.split('2')[0]
|
||||||
|
|
||||||
|
doc = self.analyze(sentence, mode=mode)
|
||||||
|
valid_tokens = self.tok_validator(doc, mode=mode)
|
||||||
|
kind = self.check_kind(doc, mode)
|
||||||
|
feasibility = self.check_feasibility(doc,valid_tokens,mode=mode)
|
||||||
|
|
||||||
|
if not feasibility['glob']:
|
||||||
|
translated = ''
|
||||||
|
info = f"failed {[k for k,v in feasibility.items() if k!='glob' and not v]}"
|
||||||
|
else:
|
||||||
|
info=''
|
||||||
|
translated = self.reorder(doc, valid_tokens=valid_tokens, direction=direction, kind=kind)
|
||||||
|
|
||||||
|
success = len(info)==0
|
||||||
|
|
||||||
|
return translated, success, info
|
||||||
@ -0,0 +1,30 @@
|
|||||||
|
sentence_kinds = ['affirmative','negative','interrogative_open','interrogative_close']
|
||||||
|
|
||||||
|
spoken_stopwords = ['il','lo','la','i','gli', 'le', 'un', 'una','uno','ce','ci','e','con']
|
||||||
|
open_question_words = ["perchè","perche","perche'",'come','quando','dove','chi','che cosa','cosa', 'quale','o','oppure']
|
||||||
|
negative_words = ['non','no']
|
||||||
|
time_terms = ['adesso', 'ormai', 'prima', 'stasera', 'sempre',
|
||||||
|
'spesso', 'tuttora', 'mai', 'presto', 'tardi', 'oggi',
|
||||||
|
'domani', 'stamani', 'recentemente', 'successivamente']#dopo,ancora, sempre,
|
||||||
|
|
||||||
|
week_days =['lunedi','martedi','mercoledi','giovedi','venerdi','sabato','domenica']
|
||||||
|
week_days +=[day.replace('i','ì') for day in week_days[:5]]
|
||||||
|
time_terms += week_days
|
||||||
|
|
||||||
|
def extend_words(words):
|
||||||
|
words += [word[0].upper()+word[1:] for word in words]
|
||||||
|
words += [word.upper() for word in words]
|
||||||
|
return words
|
||||||
|
|
||||||
|
spoken_stopwords = extend_words(spoken_stopwords)
|
||||||
|
open_question_words = extend_words(open_question_words)
|
||||||
|
negative_words = extend_words(negative_words)
|
||||||
|
time_terms = extend_words(time_terms)
|
||||||
|
|
||||||
|
|
||||||
|
rules = {
|
||||||
|
'spoken' : {
|
||||||
|
'max_tokens':15,'valid_tokens':8, 'max_verb_terms':4,'max_subject_terms':4,'max_time_terms':4
|
||||||
|
},
|
||||||
|
'gloss' : {'subject':2,'verb':2}
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue