fix: database.py get_db + crawl_today.py内容提取+作者入库修复
- 重写database.py为PostgreSQL兼容版本(get_db函数) - 修复crawl_today.py内容提取(使用>>>定位避免噪音) - 添加yichens_users作者入库支持 - 修复作者提取逻辑(跳过电话号码取<b>标签用户名) - 添加author_id字段提取
This commit is contained in:
parent
4282eac809
commit
76d87adadb
65
database.py
65
database.py
|
|
@ -1,57 +1,42 @@
|
||||||
"""MySQL 数据库连接池模块"""
|
"""PostgreSQL database connection module"""
|
||||||
import mysql.connector
|
import psycopg2
|
||||||
from mysql.connector import pooling
|
import os
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
|
|
||||||
class Database:
|
DB_CONFIG = {
|
||||||
_pool = None
|
'host': os.environ.get('DB_HOST', 'pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com'),
|
||||||
|
'port': int(os.environ.get('DB_PORT', 5432)),
|
||||||
@classmethod
|
'user': os.environ.get('DB_USER', 'coolbot'),
|
||||||
def get_pool(cls):
|
'password': os.environ.get('DB_PASSWORD', 'Coolbot123'),
|
||||||
if cls._pool is None:
|
'database': os.environ.get('DB_NAME', 'coolbot_data'),
|
||||||
cls._pool = pooling.MySQLConnectionPool(
|
}
|
||||||
pool_name="coolbot_pool",
|
|
||||||
pool_size=5,
|
|
||||||
host="127.0.0.1",
|
|
||||||
port=3306,
|
|
||||||
user="root",
|
|
||||||
password="Coolbot123",
|
|
||||||
database="coolbot_data",
|
|
||||||
charset="utf8mb4"
|
|
||||||
)
|
|
||||||
return cls._pool
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def get_connection(self):
|
def get_db():
|
||||||
conn = self.get_pool().get_connection()
|
"""Database connection context manager"""
|
||||||
|
conn = None
|
||||||
try:
|
try:
|
||||||
|
conn = psycopg2.connect(**DB_CONFIG)
|
||||||
yield conn
|
yield conn
|
||||||
finally:
|
|
||||||
conn.close()
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def get_cursor(self, dictionary=True):
|
|
||||||
with self.get_connection() as conn:
|
|
||||||
cursor = conn.cursor(dictionary=dictionary)
|
|
||||||
try:
|
|
||||||
yield cursor
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
if conn:
|
||||||
conn.rollback()
|
conn.rollback()
|
||||||
raise e
|
raise e
|
||||||
finally:
|
finally:
|
||||||
cursor.close()
|
if conn:
|
||||||
|
conn.close()
|
||||||
db = Database()
|
|
||||||
|
|
||||||
def test_connection():
|
def test_connection():
|
||||||
"""测试数据库连接"""
|
|
||||||
try:
|
try:
|
||||||
with db.get_cursor() as cursor:
|
with get_db() as conn:
|
||||||
cursor.execute("SELECT 1 as test")
|
cur = conn.cursor()
|
||||||
result = cursor.fetchone()
|
cur.execute("SELECT version()")
|
||||||
print(f"✅ 数据库连接成功: {result}")
|
print(f"DB OK: {str(cur.fetchone()[0])[:50]}")
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"❌ 数据库连接失败: {e}")
|
print(f"DB ERROR: {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
test_connection()
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue