seminar_extraction/EntityTagging.py at master · AlexJBanks/seminar_extraction · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
import re
from os import listdir
from os.path import isfile, join

import nltk
# nltk.download()


def tag_times(full_email):
    stime = None
    etime = None
    if "Time:" in full_email:
        headtimes = get_times(full_email.split("Time:")[1].split("\n")[0])
        if len(headtimes) > 0:
            stime = min(headtimes)
            if len(headtimes) > 1:
                etime = max(headtimes)

    if etime is None and "Abstract:" in full_email:
        bodytimes = get_times(full_email.split("Abstract:")[1])
        if stime is None and len(bodytimes) > 0:
            stime = min(bodytimes)
        if len(bodytimes) > 0 and max(bodytimes) != stime:
            etime = max(bodytimes)

    if stime is None:
        return full_email

    alltimes = get_times(full_email)

    for time in alltimes[stime]:
        full_email = tag_element(full_email, time, "stime")
    if etime is not None:
        for time in alltimes[etime]:
            full_email = tag_element(full_email, time, "etime")

    return full_email


def get_times(text):
    time_exp = r"(?i)(?P<hrs>\d{1,2})(?::(?P<min0>\d{2}) ?(?P<ap0>[ap]).?m|:(?P<min1>\d{2})| ?(?P<ap1>[ap]).?m)"
    time_iter = re.compile(time_exp).finditer(text)

    times = {}
    for time_obj in enumerate(time_iter):

        valid = True
        time = int(time_obj[1].group('hrs')) * 100

        if (time_obj[1].group('ap0') is not None and time_obj[1].group('ap0').lower() == 'p') \
                or (time_obj[1].group('ap1') is not None and time_obj[1].group('ap1').lower() == 'p'):
            time = time + 1200

        if time > 2400:
            valid = False

        mins = 0
        if time_obj[1].group('min0') is not None:
            mins = int(time_obj[1].group('min0'))
        if time_obj[1].group('min1') is not None:
            mins = int(time_obj[1].group('min1'))

        if mins > 59 or mins < 0:
            valid = False
        time = time + mins

        if valid:
            times.setdefault(time, set())
            times[time].add(time_obj[1][0])

    return times


def tag_para_sent(full_email):
    para_exp = r"(?m)^\s+?([A-Z0-9].+?(?:\n.+?)*?[.!?])\s+?$"
    list_of_sents = []
    if "Abstract:" in full_email:
        for sent in nltk.sent_tokenize(full_email.split("Abstract:")[1]):
            list_of_sents.append(sent)

        for paragraph in re.compile(para_exp).findall(full_email.split("Abstract:")[1]):
            full_email = tag_element(full_email, paragraph, "paragraph")

        for sent in list_of_sents:
            full_email = tag_element(full_email, sent.strip()[:-1], "sentence")

    return full_email


def tag_loc(full_email):
    raw_loc = open('data/locations.txt').read()
    all_loc = raw_loc.split('\n')
    if 'Place:' in full_email:
        loc = full_email.split('Place:')[1].split('\n')[0].strip()
        full_email = tag_element(full_email, loc, 'location')
        return full_email

    for loc in all_loc:
        if loc in full_email:
            full_email = tag_element(full_email, loc, 'location')
            return full_email

    return full_email


def tag_speak(full_email):
    speak = None
    if 'Who:' in full_email:
        speak = full_email.split('Who:')[1].split('\n')[0].split(',')[0].split('/')[0].strip()
    if 'WHO:' in full_email:
        speak = full_email.split('WHO:')[1].split('\n')[0].split(',')[0].split('/')[0].strip()
    if 'SPEAKER:' in full_email:
        speak = full_email.split('SPEAKER:')[1].split('\n')[0].split(',')[0].split('/')[0].strip()
    if speak is not None:
        full_email = tag_element(full_email, speak, 'speaker')
    return full_email


def tag_element(text, element, tag):
    element = element.strip()
    tag = tag.strip()
    return text.replace(element, "<" + tag + ">" + element + "</" + tag + ">")


def tag_email(cur_email):
    full_email = open(myPath + 'untagged/' + str(cur_email)).read()

    full_email = tag_para_sent(full_email)

    full_email = tag_speak(full_email)

    full_email = tag_times(full_email)

    full_email = tag_loc(full_email)

    with open(myPath + 'tagged/' + str(cur_email), 'w') as file:
        file.write(full_email)
    return full_email


myPath = 'data/email/test/'
onlyFiles = [f for f in listdir(myPath + 'untagged/') if isfile(join(myPath + 'untagged/', f))]

for email in onlyFiles:
    print(email)
    tag_email(email)