Spaces:
Sleeping
Sleeping
Update src/streamlit_app.py
Browse filesscript_dir = os.path.dirname(os.path.abspath(__file__))
candidate = os.path.join(script_dir, fname)
- src/streamlit_app.py +214 -191
src/streamlit_app.py
CHANGED
|
@@ -11,13 +11,28 @@ from collections import Counter
|
|
| 11 |
import json
|
| 12 |
import os
|
| 13 |
from datetime import datetime, timedelta
|
| 14 |
-
import
|
| 15 |
from dotenv import load_dotenv
|
| 16 |
import traceback
|
| 17 |
import plotly.graph_objects as go
|
| 18 |
import schedule
|
| 19 |
import threading
|
| 20 |
import matplotlib.pyplot as plt
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
|
| 22 |
# ์๋ํด๋ผ์ฐ๋ ์ถ๊ฐ
|
| 23 |
try:
|
|
@@ -40,41 +55,52 @@ class SchedulerState:
|
|
| 40 |
global_scheduler_state = SchedulerState()
|
| 41 |
|
| 42 |
# API ํค ๊ด๋ฆฌ๋ฅผ ์ํ ์ธ์
์ํ ์ด๊ธฐํ
|
| 43 |
-
if '
|
| 44 |
-
st.session_state.
|
| 45 |
|
| 46 |
# ์ฌ๋ฌ ๋ฐฉ๋ฒ์ผ๋ก API ํค ๋ก๋ ์๋
|
| 47 |
load_dotenv() # .env ํ์ผ์์ ๋ก๋ ์๋
|
| 48 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
# 1. ํ๊ฒฝ ๋ณ์์์ API ํค ํ์ธ
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
|
| 54 |
-
# 2. Streamlit secrets์์ API ํค ํ์ธ
|
| 55 |
-
if not st.session_state.
|
| 56 |
try:
|
| 57 |
if 'OPENAI_API_KEY' in st.secrets:
|
| 58 |
-
st.session_state.
|
| 59 |
-
openai.api_key = st.session_state.openai_api_key
|
| 60 |
except Exception as e:
|
| 61 |
pass # secrets ํ์ผ์ด ์์ด๋ ์ค๋ฅ ๋ฐ์ํ์ง ์์
|
| 62 |
|
| 63 |
-
#
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
nltk.data.path.insert(0, nltk_data_dir) # ์ด ๊ฒฝ๋ก๋ฅผ ์ฐ์ ๊ฒ์ํ๋๋ก ์ค์
|
| 67 |
|
| 68 |
-
# ํ์ํ NLTK ๋ฐ์ดํฐ
|
| 69 |
try:
|
| 70 |
nltk.data.find('tokenizers/punkt')
|
| 71 |
except LookupError:
|
| 72 |
-
nltk.download('punkt', download_dir=
|
| 73 |
|
| 74 |
try:
|
| 75 |
nltk.data.find('corpora/stopwords')
|
| 76 |
except LookupError:
|
| 77 |
-
nltk.download('stopwords', download_dir=
|
| 78 |
|
| 79 |
# ํ์ด์ง ์ค์
|
| 80 |
st.set_page_config(page_title="๋ด์ค ๊ธฐ์ฌ ๋๊ตฌ", page_icon="๐ฐ", layout="wide")
|
|
@@ -90,9 +116,12 @@ with st.sidebar:
|
|
| 90 |
st.divider()
|
| 91 |
api_key = st.text_input("OpenAI API ํค ์
๋ ฅ", type="password")
|
| 92 |
if api_key:
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
|
|
|
|
|
|
|
|
|
| 96 |
|
| 97 |
# ์ ์ฅ๋ ๊ธฐ์ฌ๋ฅผ ๋ถ๋ฌ์ค๋ ํจ์
|
| 98 |
def load_saved_articles():
|
|
@@ -112,16 +141,21 @@ def crawl_naver_news(keyword, num_articles=5):
|
|
| 112 |
"""
|
| 113 |
๋ค์ด๋ฒ ๋ด์ค ๊ธฐ์ฌ๋ฅผ ์์งํ๋ ํจ์
|
| 114 |
"""
|
|
|
|
| 115 |
url = f"https://search.naver.com/search.naver?where=news&query={keyword}"
|
| 116 |
results = []
|
| 117 |
|
| 118 |
try:
|
| 119 |
# ํ์ด์ง ์์ฒญ
|
|
|
|
| 120 |
response = requests.get(url)
|
|
|
|
|
|
|
| 121 |
soup = BeautifulSoup(response.text, 'html.parser')
|
| 122 |
|
| 123 |
# ๋ด์ค ์์ดํ
์ฐพ๊ธฐ
|
| 124 |
news_items = soup.select('div.sds-comps-base-layout.sds-comps-full-layout')
|
|
|
|
| 125 |
|
| 126 |
# ๊ฐ ๋ด์ค ์์ดํ
์์ ์ ๋ณด ์ถ์ถ
|
| 127 |
for i, item in enumerate(news_items):
|
|
@@ -156,48 +190,68 @@ def crawl_naver_news(keyword, num_articles=5):
|
|
| 156 |
'description': description,
|
| 157 |
'source': source,
|
| 158 |
'date': date,
|
| 159 |
-
'content': ""
|
| 160 |
})
|
| 161 |
|
|
|
|
|
|
|
| 162 |
except Exception as e:
|
| 163 |
-
|
| 164 |
continue
|
| 165 |
|
| 166 |
except Exception as e:
|
| 167 |
-
|
| 168 |
|
|
|
|
| 169 |
return results
|
| 170 |
|
| 171 |
# ๊ธฐ์ฌ ์๋ฌธ ๊ฐ์ ธ์ค๊ธฐ
|
| 172 |
def get_article_content(url):
|
|
|
|
| 173 |
try:
|
| 174 |
response = requests.get(url, timeout=5)
|
|
|
|
|
|
|
| 175 |
soup = BeautifulSoup(response.text, 'html.parser')
|
| 176 |
|
| 177 |
# ๋ค์ด๋ฒ ๋ด์ค ๋ณธ๋ฌธ ์ฐพ๊ธฐ
|
| 178 |
content = soup.select_one('#dic_area')
|
| 179 |
if content:
|
| 180 |
text = content.text.strip()
|
| 181 |
-
text = re.sub(r'\s+', ' ', text)
|
|
|
|
| 182 |
return text
|
| 183 |
|
| 184 |
-
# ๋ค๋ฅธ ๋ด์ค ์ฌ์ดํธ ๋ณธ๋ฌธ ์ฐพ๊ธฐ
|
| 185 |
content = soup.select_one('.article_body, .article-body, .article-content, .news-content-inner')
|
| 186 |
if content:
|
| 187 |
text = content.text.strip()
|
| 188 |
text = re.sub(r'\s+', ' ', text)
|
|
|
|
| 189 |
return text
|
| 190 |
|
|
|
|
| 191 |
return "๋ณธ๋ฌธ์ ๊ฐ์ ธ์ฌ ์ ์์ต๋๋ค."
|
| 192 |
except Exception as e:
|
|
|
|
| 193 |
return f"์ค๋ฅ ๋ฐ์: {str(e)}"
|
| 194 |
|
| 195 |
-
# NLTK๋ฅผ ์ด์ฉํ ํค์๋ ๋ถ์
|
| 196 |
def analyze_keywords(text, top_n=10):
|
| 197 |
-
# ํ๊ตญ์ด ๋ถ์ฉ์ด ๋ชฉ๋ก
|
| 198 |
korean_stopwords = ['์ด', '๊ทธ', '์ ', '๊ฒ', '๋ฐ', '๋ฑ', '๋ฅผ', '์', '์', '์์', '์', '์ผ๋ก', '๋ก']
|
| 199 |
|
| 200 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 201 |
tokens = [word for word in tokens if word.isalnum() and len(word) > 1 and word not in korean_stopwords]
|
| 202 |
|
| 203 |
word_count = Counter(tokens)
|
|
@@ -274,12 +328,15 @@ def generate_wordcloud(keywords_dict):
|
|
| 274 |
).generate_from_frequencies(keywords_dict)
|
| 275 |
|
| 276 |
try:
|
| 277 |
-
|
|
|
|
|
|
|
| 278 |
|
| 279 |
font_path = None
|
| 280 |
for path in possible_font_paths:
|
| 281 |
-
|
| 282 |
-
|
|
|
|
| 283 |
break
|
| 284 |
|
| 285 |
if font_path:
|
|
@@ -325,36 +382,55 @@ def analyze_news_content(news_df):
|
|
| 325 |
results['top_keywords'] = []
|
| 326 |
return results
|
| 327 |
|
| 328 |
-
# OpenAI API๋ฅผ ์ด์ฉํ ์ ๊ธฐ์ฌ ์์ฑ (
|
| 329 |
def generate_article(original_content, prompt_text):
|
| 330 |
try:
|
| 331 |
-
if not st.session_state.
|
| 332 |
return "OpenAI API ํค๊ฐ ์ค์ ๋์ง ์์์ต๋๋ค."
|
| 333 |
|
| 334 |
-
response =
|
| 335 |
-
model="gpt-4
|
| 336 |
messages=[
|
| 337 |
{"role": "system", "content": "๋น์ ์ ์ ๋ฌธ์ ์ธ ๋ด์ค ๊ธฐ์์
๋๋ค. ์ฃผ์ด์ง ๋ด์ฉ์ ๋ฐํ์ผ๋ก ์๋ก์ด ๊ธฐ์ฌ๋ฅผ ์์ฑํด์ฃผ์ธ์."},
|
| 338 |
{"role": "user", "content": f"๋ค์ ๋ด์ฉ์ ๋ฐํ์ผ๋ก {prompt_text}\n\n{original_content[:1000]}"}
|
| 339 |
],
|
| 340 |
max_tokens=2000
|
| 341 |
)
|
| 342 |
-
return response.choices[0].message
|
| 343 |
except Exception as e:
|
| 344 |
return f"๊ธฐ์ฌ ์์ฑ ์ค๋ฅ: {str(e)}"
|
| 345 |
|
| 346 |
-
# OpenAI API๋ฅผ ์ด์ฉํ ์ด๋ฏธ์ง ์์ฑ (
|
| 347 |
def generate_image(prompt):
|
| 348 |
try:
|
| 349 |
-
if not st.session_state.
|
| 350 |
return "OpenAI API ํค๊ฐ ์ค์ ๋์ง ์์์ต๋๋ค."
|
| 351 |
|
| 352 |
-
|
|
|
|
|
|
|
| 353 |
prompt=prompt,
|
| 354 |
-
n=1,
|
| 355 |
size="1024x1024"
|
| 356 |
)
|
| 357 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 358 |
except Exception as e:
|
| 359 |
return f"์ด๋ฏธ์ง ์์ฑ ์ค๋ฅ: {str(e)}"
|
| 360 |
|
|
@@ -376,11 +452,14 @@ def run_scheduled_task():
|
|
| 376 |
traceback.print_exc()
|
| 377 |
|
| 378 |
def perform_news_task(task_type, keyword, num_articles, file_prefix):
|
|
|
|
| 379 |
try:
|
| 380 |
articles = crawl_naver_news(keyword, num_articles)
|
|
|
|
| 381 |
|
| 382 |
# ๊ธฐ์ฌ ๋ด์ฉ ๊ฐ์ ธ์ค๊ธฐ
|
| 383 |
-
for article in articles:
|
|
|
|
| 384 |
article['content'] = get_article_content(article['link'])
|
| 385 |
time.sleep(0.5) # ์๋ฒ ๋ถํ ๋ฐฉ์ง
|
| 386 |
|
|
@@ -392,10 +471,12 @@ def perform_news_task(task_type, keyword, num_articles, file_prefix):
|
|
| 392 |
with open(filename, 'w', encoding='utf-8') as f:
|
| 393 |
json.dump(articles, f, ensure_ascii=False, indent=2)
|
| 394 |
|
|
|
|
|
|
|
| 395 |
global_scheduler_state.last_run = datetime.now()
|
| 396 |
print(f"{datetime.now()} - {task_type} ๋ด์ค ๊ธฐ์ฌ ์์ง ์๋ฃ: {keyword}")
|
| 397 |
|
| 398 |
-
# ์ ์ญ ์ํ์ ์์ง ๊ฒฐ๊ณผ๋ฅผ ์ ์ฅ
|
| 399 |
result_item = {
|
| 400 |
'task_type': task_type,
|
| 401 |
'keyword': keyword,
|
|
@@ -406,7 +487,7 @@ def perform_news_task(task_type, keyword, num_articles, file_prefix):
|
|
| 406 |
global_scheduler_state.scheduled_results.append(result_item)
|
| 407 |
|
| 408 |
except Exception as e:
|
| 409 |
-
|
| 410 |
traceback.print_exc()
|
| 411 |
|
| 412 |
def start_scheduler(daily_tasks, interval_tasks):
|
|
@@ -563,9 +644,25 @@ elif menu == "๊ธฐ์ฌ ๋ถ์ํ๊ธฐ":
|
|
| 563 |
with keyword_tab1:
|
| 564 |
keywords = analyze_keywords(selected_article['content'])
|
| 565 |
|
| 566 |
-
# ์๊ฐํ
|
| 567 |
df = pd.DataFrame(keywords, columns=['๋จ์ด', '๋น๋์'])
|
| 568 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 569 |
|
| 570 |
st.write("**์ฃผ์ ํค์๋:**")
|
| 571 |
for word, count in keywords:
|
|
@@ -595,7 +692,14 @@ elif menu == "๊ธฐ์ฌ ๋ถ์ํ๊ธฐ":
|
|
| 595 |
# ํ
์คํธ ํต๊ณ ๊ณ์ฐ
|
| 596 |
word_count = len(re.findall(r'\b\w+\b', content))
|
| 597 |
char_count = len(content)
|
| 598 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 599 |
avg_word_length = sum(len(word) for word in re.findall(r'\b\w+\b', content)) / word_count if word_count > 0 else 0
|
| 600 |
avg_sentence_length = word_count / sentence_count if sentence_count > 0 else 0
|
| 601 |
|
|
@@ -615,136 +719,50 @@ elif menu == "๊ธฐ์ฌ ๋ถ์ํ๊ธฐ":
|
|
| 615 |
with col2:
|
| 616 |
st.metric("ํ๊ท ๋ฌธ์ฅ ๊ธธ์ด", f"{avg_sentence_length:.1f}๋จ์ด")
|
| 617 |
|
| 618 |
-
# ํ
์คํธ ๋ณต์ก์ฑ ์ ์
|
| 619 |
complexity_score = min(10, (avg_sentence_length / 10) * 5 + (avg_word_length / 5) * 5)
|
| 620 |
st.progress(complexity_score / 10)
|
| 621 |
st.write(f"ํ
์คํธ ๋ณต์ก์ฑ ์ ์: {complexity_score:.1f}/10")
|
| 622 |
-
|
| 623 |
-
#
|
| 624 |
-
st.
|
| 625 |
-
try:
|
| 626 |
-
# KoNLPy ์ค์น ํ์ธ
|
| 627 |
-
try:
|
| 628 |
-
from konlpy.tag import Okt
|
| 629 |
-
konlpy_installed = True
|
| 630 |
-
except ImportError:
|
| 631 |
-
konlpy_installed = False
|
| 632 |
-
st.warning("ํ๊ตญ์ด ํํ์ ๋ถ์์ ์ํด KoNLPy๋ฅผ ์ค์นํด์ฃผ์ธ์: pip install konlpy")
|
| 633 |
-
|
| 634 |
-
# ์์ด POS tagger ์ค๋น
|
| 635 |
-
from nltk import pos_tag
|
| 636 |
-
try:
|
| 637 |
-
nltk.data.find('taggers/averaged_perceptron_tagger')
|
| 638 |
-
except LookupError:
|
| 639 |
-
nltk.download('averaged_perceptron_tagger', download_dir=nltk_data_dir)
|
| 640 |
-
|
| 641 |
-
# ์ธ์ด ๊ฐ์ง (๊ฐ๋จํ ๋ฐฉ์)
|
| 642 |
-
is_korean = bool(re.search(r'[๊ฐ-ํฃ]', content))
|
| 643 |
-
|
| 644 |
-
if is_korean and konlpy_installed:
|
| 645 |
-
# ํ๊ตญ์ด ํํ์ ๋ถ์
|
| 646 |
-
okt = Okt()
|
| 647 |
-
tagged = okt.pos(content)
|
| 648 |
-
|
| 649 |
-
# ํ๊ตญ์ด ํ์ฌ ๋งคํ
|
| 650 |
-
pos_dict = {
|
| 651 |
-
'Noun': '๋ช
์ฌ', 'NNG': '๋ช
์ฌ', 'NNP': '๊ณ ์ ๋ช
์ฌ',
|
| 652 |
-
'Verb': '๋์ฌ', 'VV': '๋์ฌ', 'VA': 'ํ์ฉ์ฌ',
|
| 653 |
-
'Adjective': 'ํ์ฉ์ฌ',
|
| 654 |
-
'Adverb': '๋ถ์ฌ',
|
| 655 |
-
'Josa': '์กฐ์ฌ', 'Punctuation': '๊ตฌ๋์ ',
|
| 656 |
-
'Determiner': '๊ดํ์ฌ', 'Exclamation': '๊ฐํ์ฌ'
|
| 657 |
-
}
|
| 658 |
-
|
| 659 |
-
pos_counts = {'๋ช
์ฌ': 0, '๋์ฌ': 0, 'ํ์ฉ์ฌ': 0, '๋ถ์ฌ': 0, '์กฐ์ฌ': 0, '๊ตฌ๋์ ': 0, '๊ดํ์ฌ': 0, '๊ฐํ์ฌ': 0, '๊ธฐํ': 0}
|
| 660 |
-
|
| 661 |
-
for _, pos in tagged:
|
| 662 |
-
if pos in pos_dict:
|
| 663 |
-
pos_counts[pos_dict[pos]] += 1
|
| 664 |
-
elif pos.startswith('N'): # ๊ธฐํ ๋ช
์ฌ๋ฅ
|
| 665 |
-
pos_counts['๋ช
์ฌ'] += 1
|
| 666 |
-
elif pos.startswith('V'): # ๊ธฐํ ๋์ฌ๋ฅ
|
| 667 |
-
pos_counts['๋์ฌ'] += 1
|
| 668 |
-
else:
|
| 669 |
-
pos_counts['๊ธฐํ'] += 1
|
| 670 |
-
|
| 671 |
-
else:
|
| 672 |
-
# ์์ด POS ํ๊น
|
| 673 |
-
tokens = word_tokenize(content.lower())
|
| 674 |
-
tagged = pos_tag(tokens)
|
| 675 |
-
|
| 676 |
-
# ์์ด ํ์ฌ ๋งคํ
|
| 677 |
-
pos_dict = {
|
| 678 |
-
'NN': '๋ช
์ฌ', 'NNS': '๋ช
์ฌ', 'NNP': '๊ณ ์ ๋ช
์ฌ', 'NNPS': '๊ณ ์ ๋ช
์ฌ',
|
| 679 |
-
'VB': '๋์ฌ', 'VBD': '๋์ฌ', 'VBG': '๋์ฌ', 'VBN': '๋์ฌ', 'VBP': '๋์ฌ', 'VBZ': '๋์ฌ',
|
| 680 |
-
'JJ': 'ํ์ฉ์ฌ', 'JJR': 'ํ์ฉ์ฌ', 'JJS': 'ํ์ฉ์ฌ',
|
| 681 |
-
'RB': '๋ถ์ฌ', 'RBR': '๋ถ์ฌ', 'RBS': '๋ถ์ฌ'
|
| 682 |
-
}
|
| 683 |
-
|
| 684 |
-
pos_counts = {'๋ช
์ฌ': 0, '๋์ฌ': 0, 'ํ์ฉ์ฌ': 0, '๋ถ์ฌ': 0, '๊ธฐํ': 0}
|
| 685 |
-
|
| 686 |
-
for _, pos in tagged:
|
| 687 |
-
if pos in pos_dict:
|
| 688 |
-
pos_counts[pos_dict[pos]] += 1
|
| 689 |
-
else:
|
| 690 |
-
pos_counts['๊ธฐํ'] += 1
|
| 691 |
-
|
| 692 |
-
# ๊ฒฐ๊ณผ ์๊ฐํ
|
| 693 |
-
pos_df = pd.DataFrame({
|
| 694 |
-
'ํ์ฌ': list(pos_counts.keys()),
|
| 695 |
-
'๋น๋': list(pos_counts.values())
|
| 696 |
-
})
|
| 697 |
-
|
| 698 |
-
st.bar_chart(pos_df.set_index('ํ์ฌ'))
|
| 699 |
-
|
| 700 |
-
if is_korean:
|
| 701 |
-
st.info("ํ๊ตญ์ด ํ
์คํธ๊ฐ ๊ฐ์ง๋์์ต๋๋ค.")
|
| 702 |
-
else:
|
| 703 |
-
st.info("์์ด ํ
์คํธ๊ฐ ๊ฐ์ง๋์์ต๋๋ค.")
|
| 704 |
-
except Exception as e:
|
| 705 |
-
st.error(f"ํ์ฌ ๋ถ์ ์ค ์ค๋ฅ ๋ฐ์: {str(e)}")
|
| 706 |
-
st.error(traceback.format_exc())
|
| 707 |
|
| 708 |
elif analysis_type == "๊ฐ์ ๋ถ์":
|
| 709 |
if st.button("๊ฐ์ ๋ถ์ํ๊ธฐ"):
|
| 710 |
-
if st.session_state.
|
| 711 |
with st.spinner("๊ธฐ์ฌ์ ๊ฐ์ ์ ๋ถ์ ์ค์
๋๋ค..."):
|
| 712 |
try:
|
| 713 |
-
|
| 714 |
-
prompt = """
|
| 715 |
-
๋ค์ ๋ด์ค ๊ธฐ์ฌ์ ๊ฐ์ ๊ณผ ๋
ผ์กฐ๋ฅผ ๋ถ์ํ๊ณ , ์๋ ์์์ฒ๋ผ JSON๋ง ๋ฐํํ์ธ์.
|
| 716 |
-
๋ถํ์ํ ์ค๋ช
, ์ธ์ฌ๋ง, ๊ธฐํ ํ
์คํธ ์์ด ๋ฐ๋์ JSON๋ง ์ถ๋ ฅํ์ธ์.
|
| 717 |
-
|
| 718 |
-
์์:
|
| 719 |
-
{
|
| 720 |
-
"sentiment": "๊ธ์ ์ ",
|
| 721 |
-
"reason": "์ด ๊ธฐ์ฌ์์๋ ๊ธ์ ์ ์ธ ๋จ์ด์ ํํ์ด ๋ง์ด ์ฌ์ฉ๋์์ต๋๋ค.",
|
| 722 |
-
"keywords": [
|
| 723 |
-
{"word": "ํฌ๋ง", "score": 8},
|
| 724 |
-
{"word": "์ฑ๊ณต", "score": 7},
|
| 725 |
-
{"word": "๊ธฐ๋", "score": 6},
|
| 726 |
-
{"word": "์ฑ์ฅ", "score": 7},
|
| 727 |
-
{"word": "ํ์ ", "score": 8}
|
| 728 |
-
]
|
| 729 |
-
}
|
| 730 |
-
|
| 731 |
-
๋ถ์ํ ๊ธฐ์ฌ:
|
| 732 |
-
์ ๋ชฉ: {title}
|
| 733 |
-
๋ด์ฉ: {content}
|
| 734 |
-
"""
|
| 735 |
-
|
| 736 |
-
# ๊ฐ์ ๋ถ์
|
| 737 |
-
response = openai.ChatCompletion.create(
|
| 738 |
model="gpt-4.1-mini",
|
| 739 |
messages=[
|
| 740 |
-
{"role": "system", "content": "๋น์ ์ ํ
์คํธ์ ๊ฐ์ ๊ณผ ๋
ผ์กฐ๋ฅผ ๋ถ์ํ๋ ์ ๋ฌธ๊ฐ์
๋๋ค.
|
| 741 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 742 |
],
|
| 743 |
-
max_tokens=800
|
|
|
|
| 744 |
)
|
| 745 |
|
| 746 |
-
#
|
| 747 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 748 |
|
| 749 |
# ๊ฒฐ๊ณผ ์๊ฐํ
|
| 750 |
st.subheader("๊ฐ์ ๋ถ์ ๊ฒฐ๊ณผ")
|
|
@@ -893,9 +911,9 @@ elif menu == "๊ธฐ์ฌ ๋ถ์ํ๊ธฐ":
|
|
| 893 |
|
| 894 |
except Exception as e:
|
| 895 |
st.error(f"๊ฐ์ ๋ถ์ ์ค๋ฅ: {str(e)}")
|
| 896 |
-
st.
|
| 897 |
else:
|
| 898 |
-
st.warning("OpenAI API
|
| 899 |
|
| 900 |
elif menu == "์ ๊ธฐ์ฌ ์์ฑํ๊ธฐ":
|
| 901 |
st.header("์ ๊ธฐ์ฌ ์์ฑํ๊ธฐ")
|
|
@@ -930,7 +948,7 @@ elif menu == "์ ๊ธฐ์ฌ ์์ฑํ๊ธฐ":
|
|
| 930 |
generate_image_too = st.checkbox("๊ธฐ์ฌ ์์ฑ ํ ์ด๋ฏธ์ง๋ ํจ๊ป ์์ฑํ๊ธฐ", value=True)
|
| 931 |
|
| 932 |
if st.button("์ ๊ธฐ์ฌ ์์ฑํ๊ธฐ"):
|
| 933 |
-
if st.session_state.
|
| 934 |
with st.spinner("๊ธฐ์ฌ๋ฅผ ์์ฑ ์ค์
๋๋ค..."):
|
| 935 |
new_article = generate_article(selected_article['content'], prompt_text)
|
| 936 |
|
|
@@ -940,7 +958,6 @@ elif menu == "์ ๊ธฐ์ฌ ์์ฑํ๊ธฐ":
|
|
| 940 |
# ์ด๋ฏธ์ง ์์ฑํ๊ธฐ (์ต์
์ด ์ ํ๋ ๊ฒฝ์ฐ)
|
| 941 |
if generate_image_too:
|
| 942 |
with st.spinner("๊ธฐ์ฌ ๊ด๋ จ ์ด๋ฏธ์ง๋ฅผ ์์ฑ ์ค์
๋๋ค..."):
|
| 943 |
-
# ์ด๋ฏธ์ง ์์ฑ ํ๋กฌํํธ ์ค๋น
|
| 944 |
image_prompt = f"""์ ๋ฌธ๊ธฐ์ฌ ์ ๋ชฉ "{selected_article['title']}" ์ ๋ณด๊ณ ์ด๋ฏธ์ง๋ฅผ ๋ง๋ค์ด์ค
|
| 945 |
์ด๋ฏธ์ง์๋ ๋ค์ ์์๊ฐ ํฌํจ๋์ด์ผ ํฉ๋๋ค:
|
| 946 |
- ๊ธฐ์ฌ๋ฅผ ์ดํดํ ์ ์๋ ๋์
|
|
@@ -949,13 +966,13 @@ elif menu == "์ ๊ธฐ์ฌ ์์ฑํ๊ธฐ":
|
|
| 949 |
"""
|
| 950 |
|
| 951 |
# ์ด๋ฏธ์ง ์์ฑ
|
| 952 |
-
|
| 953 |
|
| 954 |
-
if
|
| 955 |
st.subheader("์์ฑ๋ ์ด๋ฏธ์ง:")
|
| 956 |
-
st.image(
|
| 957 |
else:
|
| 958 |
-
st.error(
|
| 959 |
|
| 960 |
# ์์ฑ๋ ๊ธฐ์ฌ ์ ์ฅ ์ต์
|
| 961 |
if st.button("์์ฑ๋ ๊ธฐ์ฌ ์ ์ฅ"):
|
|
@@ -973,8 +990,6 @@ elif menu == "์ ๊ธฐ์ฌ ์์ฑํ๊ธฐ":
|
|
| 973 |
else:
|
| 974 |
st.warning("OpenAI API ํค๋ฅผ ์ฌ์ด๋๋ฐ์์ ์ค์ ํด์ฃผ์ธ์.")
|
| 975 |
|
| 976 |
-
|
| 977 |
-
|
| 978 |
elif menu == "๋ด์ค ๊ธฐ์ฌ ์์ฝํ๊ธฐ":
|
| 979 |
st.header("๋ด์ค ๊ธฐ์ฌ ์์ฝํ๊ธฐ")
|
| 980 |
|
|
@@ -1061,6 +1076,30 @@ elif menu == "๋ด์ค ๊ธฐ์ฌ ์์ฝํ๊ธฐ":
|
|
| 1061 |
with tab3:
|
| 1062 |
st.subheader("์ค์ผ์ค๋ฌ ์ ์ด ๋ฐ ์ํ")
|
| 1063 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1064 |
col1, col2 = st.columns(2)
|
| 1065 |
|
| 1066 |
with col1:
|
|
@@ -1144,20 +1183,4 @@ elif menu == "๋ด์ค ๊ธฐ์ฌ ์์ฝํ๊ธฐ":
|
|
| 1144 |
|
| 1145 |
# ํธํฐ
|
| 1146 |
st.markdown("---")
|
| 1147 |
-
st.markdown("ยฉ ๋ด์ค ๊ธฐ์ฌ ๋๊ตฌ @conanssam")
|
| 1148 |
-
|
| 1149 |
-
def extract_json_from_response(response_text):
|
| 1150 |
-
# JSON ๊ฐ์ฒด ๋ถ๋ถ๋ง ์ถ์ถ (๊ฐ์ฅ ๋จผ์ ๋์ค๋ ์ค๊ดํธ ์)
|
| 1151 |
-
match = re.search(r'\{.*\}', response_text, re.DOTALL)
|
| 1152 |
-
if match:
|
| 1153 |
-
json_str = match.group(0)
|
| 1154 |
-
try:
|
| 1155 |
-
return json.loads(json_str)
|
| 1156 |
-
except Exception as e:
|
| 1157 |
-
st.error(f"JSON ํ์ฑ ์ค๋ฅ: {str(e)}")
|
| 1158 |
-
st.code(json_str)
|
| 1159 |
-
return None
|
| 1160 |
-
else:
|
| 1161 |
-
st.error("์๋ต์์ JSON์ ์ฐพ์ ์ ์์ต๋๋ค.")
|
| 1162 |
-
st.code(response_text)
|
| 1163 |
-
return None
|
|
|
|
| 11 |
import json
|
| 12 |
import os
|
| 13 |
from datetime import datetime, timedelta
|
| 14 |
+
from openai import OpenAI # ์๋ก์ด import ๋ฐฉ์
|
| 15 |
from dotenv import load_dotenv
|
| 16 |
import traceback
|
| 17 |
import plotly.graph_objects as go
|
| 18 |
import schedule
|
| 19 |
import threading
|
| 20 |
import matplotlib.pyplot as plt
|
| 21 |
+
import kss # KoNLPy ๋์ KSS ์ฌ์ฉ
|
| 22 |
+
from PIL import Image
|
| 23 |
+
import base64
|
| 24 |
+
from io import BytesIO
|
| 25 |
+
import logging
|
| 26 |
+
|
| 27 |
+
# ๋ก๊น
์ค์
|
| 28 |
+
logging.basicConfig(
|
| 29 |
+
level=logging.INFO,
|
| 30 |
+
format='%(asctime)s - %(levelname)s - %(message)s',
|
| 31 |
+
handlers=[
|
| 32 |
+
logging.StreamHandler(),
|
| 33 |
+
logging.FileHandler('/tmp/crawler.log')
|
| 34 |
+
]
|
| 35 |
+
)
|
| 36 |
|
| 37 |
# ์๋ํด๋ผ์ฐ๋ ์ถ๊ฐ
|
| 38 |
try:
|
|
|
|
| 55 |
global_scheduler_state = SchedulerState()
|
| 56 |
|
| 57 |
# API ํค ๊ด๋ฆฌ๋ฅผ ์ํ ์ธ์
์ํ ์ด๊ธฐํ
|
| 58 |
+
if 'openai_client' not in st.session_state:
|
| 59 |
+
st.session_state.openai_client = None
|
| 60 |
|
| 61 |
# ์ฌ๋ฌ ๋ฐฉ๋ฒ์ผ๋ก API ํค ๋ก๋ ์๋
|
| 62 |
load_dotenv() # .env ํ์ผ์์ ๋ก๋ ์๋
|
| 63 |
|
| 64 |
+
# OpenAI ํด๋ผ์ด์ธํธ ์ด๊ธฐํ๋ฅผ ์ํ ํจ์
|
| 65 |
+
def init_openai_client(api_key=None):
|
| 66 |
+
try:
|
| 67 |
+
if api_key:
|
| 68 |
+
client = OpenAI(api_key=api_key)
|
| 69 |
+
# ๊ฐ๋จํ API ํค ์ ํจ์ฑ ๊ฒ์ฌ
|
| 70 |
+
client.models.list() # API ํค๊ฐ ์ ํจํ์ง ํ
์คํธ
|
| 71 |
+
return client
|
| 72 |
+
return None
|
| 73 |
+
except Exception as e:
|
| 74 |
+
st.error(f"API ํค ์ด๊ธฐํ ์ค๋ฅ: {str(e)}")
|
| 75 |
+
return None
|
| 76 |
+
|
| 77 |
# 1. ํ๊ฒฝ ๋ณ์์์ API ํค ํ์ธ
|
| 78 |
+
api_key = os.environ.get('OPENAI_API_KEY')
|
| 79 |
+
if api_key:
|
| 80 |
+
st.session_state.openai_client = init_openai_client(api_key)
|
| 81 |
|
| 82 |
+
# 2. Streamlit secrets์์ API ํค ํ์ธ
|
| 83 |
+
if not st.session_state.openai_client:
|
| 84 |
try:
|
| 85 |
if 'OPENAI_API_KEY' in st.secrets:
|
| 86 |
+
st.session_state.openai_client = init_openai_client(st.secrets['OPENAI_API_KEY'])
|
|
|
|
| 87 |
except Exception as e:
|
| 88 |
pass # secrets ํ์ผ์ด ์์ด๋ ์ค๋ฅ ๋ฐ์ํ์ง ์์
|
| 89 |
|
| 90 |
+
# NLTK ๋ฐ์ดํฐ ๊ฒฝ๋ก ์ค์ - ํ์ฌ ์ํฌ์คํ์ด์ค์ nltk_data ์ฌ์ฉ
|
| 91 |
+
nltk_data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'nltk_data')
|
| 92 |
+
nltk.data.path.insert(0, nltk_data_path)
|
|
|
|
| 93 |
|
| 94 |
+
# ํ์ํ NLTK ๋ฐ์ดํฐ ํ์ธ
|
| 95 |
try:
|
| 96 |
nltk.data.find('tokenizers/punkt')
|
| 97 |
except LookupError:
|
| 98 |
+
nltk.download('punkt', download_dir=nltk_data_path)
|
| 99 |
|
| 100 |
try:
|
| 101 |
nltk.data.find('corpora/stopwords')
|
| 102 |
except LookupError:
|
| 103 |
+
nltk.download('stopwords', download_dir=nltk_data_path)
|
| 104 |
|
| 105 |
# ํ์ด์ง ์ค์
|
| 106 |
st.set_page_config(page_title="๋ด์ค ๊ธฐ์ฌ ๋๊ตฌ", page_icon="๐ฐ", layout="wide")
|
|
|
|
| 116 |
st.divider()
|
| 117 |
api_key = st.text_input("OpenAI API ํค ์
๋ ฅ", type="password")
|
| 118 |
if api_key:
|
| 119 |
+
client = init_openai_client(api_key)
|
| 120 |
+
if client:
|
| 121 |
+
st.session_state.openai_client = client
|
| 122 |
+
st.success("API ํค๊ฐ ์ฑ๊ณต์ ์ผ๋ก ์ค์ ๋์์ต๋๋ค!")
|
| 123 |
+
else:
|
| 124 |
+
st.error("์ ํจํ์ง ์์ API ํค์
๋๋ค.")
|
| 125 |
|
| 126 |
# ์ ์ฅ๋ ๊ธฐ์ฌ๋ฅผ ๋ถ๋ฌ์ค๋ ํจ์
|
| 127 |
def load_saved_articles():
|
|
|
|
| 141 |
"""
|
| 142 |
๋ค์ด๋ฒ ๋ด์ค ๊ธฐ์ฌ๋ฅผ ์์งํ๋ ํจ์
|
| 143 |
"""
|
| 144 |
+
logging.info(f"ํฌ๋กค๋ง ์์: ํค์๋={keyword}, ๊ธฐ์ฌ ์={num_articles}")
|
| 145 |
url = f"https://search.naver.com/search.naver?where=news&query={keyword}"
|
| 146 |
results = []
|
| 147 |
|
| 148 |
try:
|
| 149 |
# ํ์ด์ง ์์ฒญ
|
| 150 |
+
logging.info(f"์์ฒญ URL: {url}")
|
| 151 |
response = requests.get(url)
|
| 152 |
+
logging.info(f"์๋ต ์ํ ์ฝ๋: {response.status_code}")
|
| 153 |
+
|
| 154 |
soup = BeautifulSoup(response.text, 'html.parser')
|
| 155 |
|
| 156 |
# ๋ด์ค ์์ดํ
์ฐพ๊ธฐ
|
| 157 |
news_items = soup.select('div.sds-comps-base-layout.sds-comps-full-layout')
|
| 158 |
+
logging.info(f"์ฐพ์ ๋ด์ค ์์ดํ
์: {len(news_items)}")
|
| 159 |
|
| 160 |
# ๊ฐ ๋ด์ค ์์ดํ
์์ ์ ๋ณด ์ถ์ถ
|
| 161 |
for i, item in enumerate(news_items):
|
|
|
|
| 190 |
'description': description,
|
| 191 |
'source': source,
|
| 192 |
'date': date,
|
| 193 |
+
'content': ""
|
| 194 |
})
|
| 195 |
|
| 196 |
+
logging.info(f"๊ธฐ์ฌ ์ถ์ถ ์ฑ๊ณต: {title}")
|
| 197 |
+
|
| 198 |
except Exception as e:
|
| 199 |
+
logging.error(f"๊ธฐ์ฌ ์ ๋ณด ์ถ์ถ ์ค ์ค๋ฅ ๋ฐ์: {str(e)}", exc_info=True)
|
| 200 |
continue
|
| 201 |
|
| 202 |
except Exception as e:
|
| 203 |
+
logging.error(f"ํ์ด์ง ์์ฒญ ์ค ์ค๋ฅ ๋ฐ์: {str(e)}", exc_info=True)
|
| 204 |
|
| 205 |
+
logging.info(f"ํฌ๋กค๋ง ์๋ฃ: {len(results)}๊ฐ ๊ธฐ์ฌ ์์ง")
|
| 206 |
return results
|
| 207 |
|
| 208 |
# ๊ธฐ์ฌ ์๋ฌธ ๊ฐ์ ธ์ค๊ธฐ
|
| 209 |
def get_article_content(url):
|
| 210 |
+
logging.info(f"๊ธฐ์ฌ ์๋ฌธ ๊ฐ์ ธ์ค๊ธฐ ์์: {url}")
|
| 211 |
try:
|
| 212 |
response = requests.get(url, timeout=5)
|
| 213 |
+
logging.info(f"์๋ฌธ ์์ฒญ ์ํ ์ฝ๋: {response.status_code}")
|
| 214 |
+
|
| 215 |
soup = BeautifulSoup(response.text, 'html.parser')
|
| 216 |
|
| 217 |
# ๋ค์ด๋ฒ ๋ด์ค ๋ณธ๋ฌธ ์ฐพ๊ธฐ
|
| 218 |
content = soup.select_one('#dic_area')
|
| 219 |
if content:
|
| 220 |
text = content.text.strip()
|
| 221 |
+
text = re.sub(r'\s+', ' ', text)
|
| 222 |
+
logging.info("๋ค์ด๋ฒ ๋ด์ค ๋ณธ๋ฌธ ์ถ์ถ ์ฑ๊ณต")
|
| 223 |
return text
|
| 224 |
|
| 225 |
+
# ๋ค๋ฅธ ๋ด์ค ์ฌ์ดํธ ๋ณธ๋ฌธ ์ฐพ๊ธฐ
|
| 226 |
content = soup.select_one('.article_body, .article-body, .article-content, .news-content-inner')
|
| 227 |
if content:
|
| 228 |
text = content.text.strip()
|
| 229 |
text = re.sub(r'\s+', ' ', text)
|
| 230 |
+
logging.info("์ผ๋ฐ ๋ด์ค ๋ณธ๋ฌธ ์ถ์ถ ์ฑ๊ณต")
|
| 231 |
return text
|
| 232 |
|
| 233 |
+
logging.warning("๋ณธ๋ฌธ์ ์ฐพ์ ์ ์์")
|
| 234 |
return "๋ณธ๋ฌธ์ ๊ฐ์ ธ์ฌ ์ ์์ต๋๋ค."
|
| 235 |
except Exception as e:
|
| 236 |
+
logging.error(f"์๋ฌธ ๊ฐ์ ธ์ค๊ธฐ ์ค๋ฅ: {str(e)}", exc_info=True)
|
| 237 |
return f"์ค๋ฅ ๋ฐ์: {str(e)}"
|
| 238 |
|
| 239 |
+
# NLTK๋ฅผ ์ด์ฉํ ํค์๋ ๋ถ์ (KSS ํ์ฉ)
|
| 240 |
def analyze_keywords(text, top_n=10):
|
| 241 |
+
# ํ๊ตญ์ด ๋ถ์ฉ์ด ๋ชฉ๋ก
|
| 242 |
korean_stopwords = ['์ด', '๊ทธ', '์ ', '๊ฒ', '๋ฐ', '๋ฑ', '๋ฅผ', '์', '์', '์์', '์', '์ผ๋ก', '๋ก']
|
| 243 |
|
| 244 |
+
# KSS๋ฅผ ์ฌ์ฉํ ๋ฌธ์ฅ ๋ถ๋ฆฌ ๋ฐ ํ ํฐํ
|
| 245 |
+
try:
|
| 246 |
+
sentences = kss.split_sentences(text)
|
| 247 |
+
tokens = []
|
| 248 |
+
for sentence in sentences:
|
| 249 |
+
# ๊ฐ๋จํ ํ ํฐํ (๊ณต๋ฐฑ ๊ธฐ์ค)
|
| 250 |
+
tokens.extend(sentence.split())
|
| 251 |
+
except:
|
| 252 |
+
# KSS ์คํจ์ ๊ธฐ๋ณธ ํ ํฐํ
|
| 253 |
+
tokens = text.split()
|
| 254 |
+
|
| 255 |
tokens = [word for word in tokens if word.isalnum() and len(word) > 1 and word not in korean_stopwords]
|
| 256 |
|
| 257 |
word_count = Counter(tokens)
|
|
|
|
| 328 |
).generate_from_frequencies(keywords_dict)
|
| 329 |
|
| 330 |
try:
|
| 331 |
+
import os
|
| 332 |
+
script_dir = os.path.dirname(os.path.abspath(__file__))
|
| 333 |
+
possible_font_paths = ["NanumGothic.ttf", "์ด๋ฆ"]
|
| 334 |
|
| 335 |
font_path = None
|
| 336 |
for path in possible_font_paths:
|
| 337 |
+
candidate = os.path.join(script_dir, fname)
|
| 338 |
+
if os.path.exists(candidate):
|
| 339 |
+
font_path = candidate
|
| 340 |
break
|
| 341 |
|
| 342 |
if font_path:
|
|
|
|
| 382 |
results['top_keywords'] = []
|
| 383 |
return results
|
| 384 |
|
| 385 |
+
# OpenAI API๋ฅผ ์ด์ฉํ ์ ๊ธฐ์ฌ ์์ฑ (์๋ก์ด ๋ฒ์ ๋ฐฉ์)
|
| 386 |
def generate_article(original_content, prompt_text):
|
| 387 |
try:
|
| 388 |
+
if not st.session_state.openai_client:
|
| 389 |
return "OpenAI API ํค๊ฐ ์ค์ ๋์ง ์์์ต๋๋ค."
|
| 390 |
|
| 391 |
+
response = st.session_state.openai_client.chat.completions.create(
|
| 392 |
+
model="gpt-4", # ๋๋ ์ฌ์ฉ ๊ฐ๋ฅํ ์ ์ ํ ๋ชจ๋ธ
|
| 393 |
messages=[
|
| 394 |
{"role": "system", "content": "๋น์ ์ ์ ๋ฌธ์ ์ธ ๋ด์ค ๊ธฐ์์
๋๋ค. ์ฃผ์ด์ง ๋ด์ฉ์ ๋ฐํ์ผ๋ก ์๋ก์ด ๊ธฐ์ฌ๋ฅผ ์์ฑํด์ฃผ์ธ์."},
|
| 395 |
{"role": "user", "content": f"๋ค์ ๋ด์ฉ์ ๋ฐํ์ผ๋ก {prompt_text}\n\n{original_content[:1000]}"}
|
| 396 |
],
|
| 397 |
max_tokens=2000
|
| 398 |
)
|
| 399 |
+
return response.choices[0].message.content
|
| 400 |
except Exception as e:
|
| 401 |
return f"๊ธฐ์ฌ ์์ฑ ์ค๋ฅ: {str(e)}"
|
| 402 |
|
| 403 |
+
# OpenAI API๋ฅผ ์ด์ฉํ ์ด๋ฏธ์ง ์์ฑ (์๋ก์ด ๋ฒ์ ๋ฐฉ์)
|
| 404 |
def generate_image(prompt):
|
| 405 |
try:
|
| 406 |
+
if not st.session_state.openai_client:
|
| 407 |
return "OpenAI API ํค๊ฐ ์ค์ ๋์ง ์์์ต๋๋ค."
|
| 408 |
|
| 409 |
+
# GPT Image 1 ๋ชจ๋ธ๋ก ์ด๋ฏธ์ง ์์ฑ
|
| 410 |
+
result = st.session_state.openai_client.images.generate(
|
| 411 |
+
model="gpt-image-1", # ์๋ก์ด ๋ชจ๋ธ๋ช
์ฌ์ฉ
|
| 412 |
prompt=prompt,
|
|
|
|
| 413 |
size="1024x1024"
|
| 414 |
)
|
| 415 |
+
|
| 416 |
+
# base64 ์ด๋ฏธ์ง ๋ฐ์ดํฐ๋ฅผ ๋์ฝ๋ฉ
|
| 417 |
+
image_base64 = result.data[0].b64_json
|
| 418 |
+
image_bytes = base64.b64decode(image_base64)
|
| 419 |
+
|
| 420 |
+
# BytesIO ๊ฐ์ฒด๋ก ๋ณํ
|
| 421 |
+
image = BytesIO(image_bytes)
|
| 422 |
+
|
| 423 |
+
# PIL Image๋ก ๋ณํํ์ฌ ํฌ๊ธฐ ์กฐ์ (์ ํ์ฌํญ)
|
| 424 |
+
pil_image = Image.open(image)
|
| 425 |
+
pil_image = pil_image.resize((800, 800), Image.LANCZOS) # ํฌ๊ธฐ ์กฐ์
|
| 426 |
+
|
| 427 |
+
# ๋ค์ BytesIO๋ก ๋ณํ
|
| 428 |
+
output = BytesIO()
|
| 429 |
+
pil_image.save(output, format="JPEG", quality=80, optimize=True)
|
| 430 |
+
output.seek(0)
|
| 431 |
+
|
| 432 |
+
return output
|
| 433 |
+
|
| 434 |
except Exception as e:
|
| 435 |
return f"์ด๋ฏธ์ง ์์ฑ ์ค๋ฅ: {str(e)}"
|
| 436 |
|
|
|
|
| 452 |
traceback.print_exc()
|
| 453 |
|
| 454 |
def perform_news_task(task_type, keyword, num_articles, file_prefix):
|
| 455 |
+
logging.info(f"์ค์ผ์ค๋ฌ ์์
์์: {task_type}, ํค์๋={keyword}")
|
| 456 |
try:
|
| 457 |
articles = crawl_naver_news(keyword, num_articles)
|
| 458 |
+
logging.info(f"์์ง๋ ๊ธฐ์ฌ ์: {len(articles)}")
|
| 459 |
|
| 460 |
# ๊ธฐ์ฌ ๋ด์ฉ ๊ฐ์ ธ์ค๊ธฐ
|
| 461 |
+
for i, article in enumerate(articles):
|
| 462 |
+
logging.info(f"๊ธฐ์ฌ {i+1}/{len(articles)} ์๋ฌธ ๊ฐ์ ธ์ค๊ธฐ: {article['title']}")
|
| 463 |
article['content'] = get_article_content(article['link'])
|
| 464 |
time.sleep(0.5) # ์๋ฒ ๋ถํ ๋ฐฉ์ง
|
| 465 |
|
|
|
|
| 471 |
with open(filename, 'w', encoding='utf-8') as f:
|
| 472 |
json.dump(articles, f, ensure_ascii=False, indent=2)
|
| 473 |
|
| 474 |
+
logging.info(f"๊ฒฐ๊ณผ ์ ์ฅ ์๋ฃ: {filename}")
|
| 475 |
+
|
| 476 |
global_scheduler_state.last_run = datetime.now()
|
| 477 |
print(f"{datetime.now()} - {task_type} ๋ด์ค ๊ธฐ์ฌ ์์ง ์๋ฃ: {keyword}")
|
| 478 |
|
| 479 |
+
# ์ ์ญ ์ํ์ ์์ง ๊ฒฐ๊ณผ๋ฅผ ์ ์ฅ
|
| 480 |
result_item = {
|
| 481 |
'task_type': task_type,
|
| 482 |
'keyword': keyword,
|
|
|
|
| 487 |
global_scheduler_state.scheduled_results.append(result_item)
|
| 488 |
|
| 489 |
except Exception as e:
|
| 490 |
+
logging.error(f"์์
์คํ ์ค ์ค๋ฅ ๋ฐ์: {str(e)}", exc_info=True)
|
| 491 |
traceback.print_exc()
|
| 492 |
|
| 493 |
def start_scheduler(daily_tasks, interval_tasks):
|
|
|
|
| 644 |
with keyword_tab1:
|
| 645 |
keywords = analyze_keywords(selected_article['content'])
|
| 646 |
|
| 647 |
+
# Plotly๋ฅผ ์ฌ์ฉํ ์๊ฐํ
|
| 648 |
df = pd.DataFrame(keywords, columns=['๋จ์ด', '๋น๋์'])
|
| 649 |
+
fig = go.Figure(data=[
|
| 650 |
+
go.Bar(
|
| 651 |
+
x=df['๋จ์ด'],
|
| 652 |
+
y=df['๋น๋์'],
|
| 653 |
+
marker_color='rgb(55, 83, 109)'
|
| 654 |
+
)
|
| 655 |
+
])
|
| 656 |
+
|
| 657 |
+
fig.update_layout(
|
| 658 |
+
title='ํค์๋ ๋น๋ ๋ถ์',
|
| 659 |
+
xaxis_title='ํค์๋',
|
| 660 |
+
yaxis_title='๋น๋์',
|
| 661 |
+
height=500,
|
| 662 |
+
margin=dict(l=50, r=50, t=80, b=50)
|
| 663 |
+
)
|
| 664 |
+
|
| 665 |
+
st.plotly_chart(fig, use_container_width=True)
|
| 666 |
|
| 667 |
st.write("**์ฃผ์ ํค์๋:**")
|
| 668 |
for word, count in keywords:
|
|
|
|
| 692 |
# ํ
์คํธ ํต๊ณ ๊ณ์ฐ
|
| 693 |
word_count = len(re.findall(r'\b\w+\b', content))
|
| 694 |
char_count = len(content)
|
| 695 |
+
try:
|
| 696 |
+
# KSS๋ก ๋ฌธ์ฅ ๋ถ๋ฆฌ
|
| 697 |
+
sentences = kss.split_sentences(content)
|
| 698 |
+
sentence_count = len(sentences)
|
| 699 |
+
except:
|
| 700 |
+
# KSS ์คํจ์ ๊ธฐ๋ณธ ๋ฌธ์ฅ ๋ถ๋ฆฌ
|
| 701 |
+
sentence_count = len(re.split(r'[.!?]+', content))
|
| 702 |
+
|
| 703 |
avg_word_length = sum(len(word) for word in re.findall(r'\b\w+\b', content)) / word_count if word_count > 0 else 0
|
| 704 |
avg_sentence_length = word_count / sentence_count if sentence_count > 0 else 0
|
| 705 |
|
|
|
|
| 719 |
with col2:
|
| 720 |
st.metric("ํ๊ท ๋ฌธ์ฅ ๊ธธ์ด", f"{avg_sentence_length:.1f}๋จ์ด")
|
| 721 |
|
| 722 |
+
# ํ
์คํธ ๋ณต์ก์ฑ ์ ์
|
| 723 |
complexity_score = min(10, (avg_sentence_length / 10) * 5 + (avg_word_length / 5) * 5)
|
| 724 |
st.progress(complexity_score / 10)
|
| 725 |
st.write(f"ํ
์คํธ ๋ณต์ก์ฑ ์ ์: {complexity_score:.1f}/10")
|
| 726 |
+
|
| 727 |
+
# ํ์ฌ ๋ถ์ ๋ถ๋ถ ์ ๊ฑฐ (KoNLPy ์์กด์ฑ ์ ๊ฑฐ)
|
| 728 |
+
st.info("์์ธ ํ์ฌ ๋ถ์์ ํ์ฌ ์ง์๋์ง ์์ต๋๋ค.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 729 |
|
| 730 |
elif analysis_type == "๊ฐ์ ๋ถ์":
|
| 731 |
if st.button("๊ฐ์ ๋ถ์ํ๊ธฐ"):
|
| 732 |
+
if st.session_state.openai_client:
|
| 733 |
with st.spinner("๊ธฐ์ฌ์ ๊ฐ์ ์ ๋ถ์ ์ค์
๋๋ค..."):
|
| 734 |
try:
|
| 735 |
+
response = st.session_state.openai_client.chat.completions.create(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 736 |
model="gpt-4.1-mini",
|
| 737 |
messages=[
|
| 738 |
+
{"role": "system", "content": """๋น์ ์ ํ
์คํธ์ ๊ฐ์ ๊ณผ ๋
ผ์กฐ๋ฅผ ๋ถ์ํ๋ ์ ๋ฌธ๊ฐ์
๋๋ค.
|
| 739 |
+
๋ค์ ๋ด์ค ๊ธฐ์ฌ์ ๊ฐ์ ๊ณผ ๋
ผ์กฐ๋ฅผ ๋ถ์ํ๊ณ , ๋ฐ๋์ ์๋ ํ์์ JSON์ผ๋ก ์๋ตํด์ฃผ์ธ์:
|
| 740 |
+
{
|
| 741 |
+
"sentiment": "๊ธ์ ์ /๋ถ์ ์ /์ค๋ฆฝ์ ",
|
| 742 |
+
"reason": "์ด์ ์ค๋ช
...",
|
| 743 |
+
"keywords": [
|
| 744 |
+
{"word": "ํค์๋1", "score": 8},
|
| 745 |
+
{"word": "ํค์๋2", "score": 7}
|
| 746 |
+
]
|
| 747 |
+
}"""},
|
| 748 |
+
{"role": "user", "content": f"๋ค์ ๋ด์ค ๊ธฐ์ฌ๋ฅผ ๋ถ์ํด ์ฃผ์ธ์:\n\n์ ๋ชฉ: {selected_article['title']}\n\n๋ด์ฉ: {selected_article['content'][:1500]}"}
|
| 749 |
],
|
| 750 |
+
max_tokens=800,
|
| 751 |
+
response_format={ "type": "json_object" } # JSON ์๋ต ํ์ ๊ฐ์
|
| 752 |
)
|
| 753 |
|
| 754 |
+
# ์๋ต ๋ด์ฉ ํ์ธ ๋ฐ ๋๋ฒ๊น
|
| 755 |
+
content = response.choices[0].message.content
|
| 756 |
+
logging.info(f"API ์๋ต: {content}")
|
| 757 |
+
|
| 758 |
+
# JSON ํ์ฑ
|
| 759 |
+
try:
|
| 760 |
+
analysis_result = json.loads(content)
|
| 761 |
+
except json.JSONDecodeError as e:
|
| 762 |
+
logging.error(f"JSON ํ์ฑ ์ค๋ฅ: {str(e)}")
|
| 763 |
+
logging.error(f"ํ์ฑ ์๋ํ ๋ด์ฉ: {content}")
|
| 764 |
+
st.error("API ์๋ต์ ํ์ฑํ๋ ์ค ์ค๋ฅ๊ฐ ๋ฐ์ํ์ต๋๋ค. ์๋ต ํ์์ด ์ฌ๋ฐ๋ฅด์ง ์์ต๋๋ค.")
|
| 765 |
+
st.stop() # return ๋์ st.stop() ์ฌ์ฉ
|
| 766 |
|
| 767 |
# ๊ฒฐ๊ณผ ์๊ฐํ
|
| 768 |
st.subheader("๊ฐ์ ๋ถ์ ๊ฒฐ๊ณผ")
|
|
|
|
| 911 |
|
| 912 |
except Exception as e:
|
| 913 |
st.error(f"๊ฐ์ ๋ถ์ ์ค๋ฅ: {str(e)}")
|
| 914 |
+
st.error(traceback.format_exc())
|
| 915 |
else:
|
| 916 |
+
st.warning("OpenAI API ํค๋ฅผ ์ฌ์ด๋๋ฐ์์ ์ค์ ํด์ฃผ์ธ์.")
|
| 917 |
|
| 918 |
elif menu == "์ ๊ธฐ์ฌ ์์ฑํ๊ธฐ":
|
| 919 |
st.header("์ ๊ธฐ์ฌ ์์ฑํ๊ธฐ")
|
|
|
|
| 948 |
generate_image_too = st.checkbox("๊ธฐ์ฌ ์์ฑ ํ ์ด๋ฏธ์ง๋ ํจ๊ป ์์ฑํ๊ธฐ", value=True)
|
| 949 |
|
| 950 |
if st.button("์ ๊ธฐ์ฌ ์์ฑํ๊ธฐ"):
|
| 951 |
+
if st.session_state.openai_client:
|
| 952 |
with st.spinner("๊ธฐ์ฌ๋ฅผ ์์ฑ ์ค์
๋๋ค..."):
|
| 953 |
new_article = generate_article(selected_article['content'], prompt_text)
|
| 954 |
|
|
|
|
| 958 |
# ์ด๋ฏธ์ง ์์ฑํ๊ธฐ (์ต์
์ด ์ ํ๋ ๊ฒฝ์ฐ)
|
| 959 |
if generate_image_too:
|
| 960 |
with st.spinner("๊ธฐ์ฌ ๊ด๋ จ ์ด๋ฏธ์ง๋ฅผ ์์ฑ ์ค์
๋๋ค..."):
|
|
|
|
| 961 |
image_prompt = f"""์ ๋ฌธ๊ธฐ์ฌ ์ ๋ชฉ "{selected_article['title']}" ์ ๋ณด๊ณ ์ด๋ฏธ์ง๋ฅผ ๋ง๋ค์ด์ค
|
| 962 |
์ด๋ฏธ์ง์๋ ๋ค์ ์์๊ฐ ํฌํจ๋์ด์ผ ํฉ๋๋ค:
|
| 963 |
- ๊ธฐ์ฌ๋ฅผ ์ดํดํ ์ ์๋ ๋์
|
|
|
|
| 966 |
"""
|
| 967 |
|
| 968 |
# ์ด๋ฏธ์ง ์์ฑ
|
| 969 |
+
image = generate_image(image_prompt)
|
| 970 |
|
| 971 |
+
if isinstance(image, BytesIO):
|
| 972 |
st.subheader("์์ฑ๋ ์ด๋ฏธ์ง:")
|
| 973 |
+
st.image(image, use_column_width=True)
|
| 974 |
else:
|
| 975 |
+
st.error(image)
|
| 976 |
|
| 977 |
# ์์ฑ๋ ๊ธฐ์ฌ ์ ์ฅ ์ต์
|
| 978 |
if st.button("์์ฑ๋ ๊ธฐ์ฌ ์ ์ฅ"):
|
|
|
|
| 990 |
else:
|
| 991 |
st.warning("OpenAI API ํค๋ฅผ ์ฌ์ด๋๋ฐ์์ ์ค์ ํด์ฃผ์ธ์.")
|
| 992 |
|
|
|
|
|
|
|
| 993 |
elif menu == "๋ด์ค ๊ธฐ์ฌ ์์ฝํ๊ธฐ":
|
| 994 |
st.header("๋ด์ค ๊ธฐ์ฌ ์์ฝํ๊ธฐ")
|
| 995 |
|
|
|
|
| 1076 |
with tab3:
|
| 1077 |
st.subheader("์ค์ผ์ค๋ฌ ์ ์ด ๋ฐ ์ํ")
|
| 1078 |
|
| 1079 |
+
# ๋ก๊ทธ ๋ทฐ์ด๋ฅผ ์๋จ์ ๋ฐฐ์น
|
| 1080 |
+
st.subheader("์ค์๊ฐ ๋ก๊ทธ")
|
| 1081 |
+
log_container = st.empty()
|
| 1082 |
+
|
| 1083 |
+
def update_logs():
|
| 1084 |
+
try:
|
| 1085 |
+
with open('/tmp/crawler.log', 'r') as f:
|
| 1086 |
+
logs = f.readlines()
|
| 1087 |
+
return ''.join(logs[-100:]) # ์ต๊ทผ 100์ค๋ง ํ์
|
| 1088 |
+
except Exception as e:
|
| 1089 |
+
return f"๋ก๊ทธ ํ์ผ์ ์ฝ์ ์ ์์ต๋๋ค: {str(e)}"
|
| 1090 |
+
|
| 1091 |
+
# ๋ก๊ทธ ์๋ ์
๋ฐ์ดํธ
|
| 1092 |
+
if st.checkbox("๋ก๊ทธ ์๋ ์
๋ฐ์ดํธ", value=True):
|
| 1093 |
+
log_content = update_logs()
|
| 1094 |
+
log_container.text_area("์ต๊ทผ ๋ก๊ทธ", value=log_content, height=400)
|
| 1095 |
+
else:
|
| 1096 |
+
if st.button("๋ก๊ทธ ์๋ก๊ณ ์นจ"):
|
| 1097 |
+
log_content = update_logs()
|
| 1098 |
+
log_container.text_area("์ต๊ทผ ๋ก๊ทธ", value=log_content, height=400)
|
| 1099 |
+
|
| 1100 |
+
st.divider()
|
| 1101 |
+
|
| 1102 |
+
# ์ค์ผ์ค๋ฌ ์ ์ด
|
| 1103 |
col1, col2 = st.columns(2)
|
| 1104 |
|
| 1105 |
with col1:
|
|
|
|
| 1183 |
|
| 1184 |
# ํธํฐ
|
| 1185 |
st.markdown("---")
|
| 1186 |
+
st.markdown("ยฉ ๋ด์ค ๊ธฐ์ฌ ๋๊ตฌ @conanssam")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|