Update app.py

app.py

@@ -8,10 +8,7 @@ from sklearn.model_selection import KFold
 from transformers import AutoTokenizer, DistilBertTokenizerFast
 # sequence tagging model + training-related 
 from transformers import DistilBertForTokenClassification, Trainer, TrainingArguments
-import numpy as np
-import pandas as pd
 import torch
-import json
 import sys
 import os
 from sklearn.metrics import classification_report
@@ -22,28 +19,22 @@ from sklearn.feature_extraction.text import TfidfTransformer
 from sklearn.feature_extraction.text import CountVectorizer
 from sklearn.pipeline import Pipeline, FeatureUnion
 import math
-from sklearn.metrics import accuracy_score
-from sklearn.metrics import precision_recall_fscore_support
-from sklearn.model_selection import train_test_split
+# from sklearn.metrics import accuracy_score
+# from sklearn.metrics import precision_recall_fscore_support
 import json
 import re
 import numpy as np 
 import pandas as pd
-import re
 import nltk
 nltk.download("punkt")
 import string
 from sklearn.model_selection import train_test_split
 from transformers import AutoTokenizer, Trainer, TrainingArguments, AutoModelForSequenceClassification, AutoConfig
-import torch
 from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
 import itertools
-import json
-import glob
 from transformers import TextClassificationPipeline, TFAutoModelForSequenceClassification, AutoTokenizer
 from transformers import pipeline
-import pickle
-import urllib.request  
+import pickle 
 import csv
 import pdfplumber
 import pathlib 
@@ -55,6 +46,7 @@ from PyPDF2 import PdfReader
 from huggingface_hub import HfApi
 import io
 from datasets import load_dataset
+import time
 
 import huggingface_hub
 from huggingface_hub import Repository
@@ -62,8 +54,8 @@ from datetime import datetime
 import pathlib as Path
 from requests import get
 import urllib.request
-import gradio as gr
-from gradio import inputs, outputs
+# import gradio as gr
+# from gradio import inputs, outputs
 from datasets import load_dataset
 from huggingface_hub import HfApi, list_models
 import os
@@ -130,7 +122,8 @@ def main():
     result1 = i.lower()
     result2 = re.sub(r'[^\w\s]','',result1)
     result.append(result2)
-    
+
+  print("--- %s seconds ---" % (time.time() - start_time))
   tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") #bert-base-uncased
   
   model_path = "checkpoint-2850"
@@ -144,6 +137,9 @@ def main():
         if lab['label'] == 'causal': #causal
             causal_sents.append(sent)
 
+  st.write('causal sentence classification finished')
+  st.write("--- %s seconds ---" % (time.time() - start_time))
+
   model_name = "distilbert-base-cased"
   tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)
   
@@ -165,7 +161,10 @@ def main():
       sentence_pred.append(k)
       class_list.append(i['word'])
       entity_list.append(i['entity_group'])
-  
+
+  st.write('causality extraction finished')
+  st.write("--- %s seconds ---" % (time.time() - start_time))
+
   # filename = 'Checkpoint-classification.sav'
   # loaded_model = pickle.load(open(filename, 'rb'))
   # loaded_vectorizer = pickle.load(open('vectorizefile_classification.pickle', 'rb'))
@@ -191,6 +190,9 @@ def main():
 
   predictions = loaded_model.predict(pad_sequences(tokenizer.texts_to_sequences(class_list),maxlen=MAX_SEQUENCE_LENGTH))
   predicted = np.argmax(predictions,axis=1)
+
+  st.write('stakeholder taxonomy finished')
+  st.write("--- %s seconds ---" % (time.time() - start_time))
   pred1 = predicted
   level0 = []
   count =0
@@ -574,4 +576,5 @@ def main():
     
 
 if __name__ == '__main__':
+    start_time = time.time()
     main()