fix: database.py get_db + crawl_today.py内容提取+作者入库修复
- 重写database.py为PostgreSQL兼容版本(get_db函数) - 修复crawl_today.py内容提取(使用>>>定位避免噪音) - 添加yichens_users作者入库支持 - 修复作者提取逻辑(跳过电话号码取<b>标签用户名) - 添加author_id字段提取
This commit is contained in:
parent
4282eac809
commit
76d87adadb
67
database.py
67
database.py
|
|
@ -1,57 +1,42 @@
|
|||
"""MySQL 数据库连接池模块"""
|
||||
import mysql.connector
|
||||
from mysql.connector import pooling
|
||||
"""PostgreSQL database connection module"""
|
||||
import psycopg2
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
|
||||
class Database:
|
||||
_pool = None
|
||||
DB_CONFIG = {
|
||||
'host': os.environ.get('DB_HOST', 'pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com'),
|
||||
'port': int(os.environ.get('DB_PORT', 5432)),
|
||||
'user': os.environ.get('DB_USER', 'coolbot'),
|
||||
'password': os.environ.get('DB_PASSWORD', 'Coolbot123'),
|
||||
'database': os.environ.get('DB_NAME', 'coolbot_data'),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def get_pool(cls):
|
||||
if cls._pool is None:
|
||||
cls._pool = pooling.MySQLConnectionPool(
|
||||
pool_name="coolbot_pool",
|
||||
pool_size=5,
|
||||
host="127.0.0.1",
|
||||
port=3306,
|
||||
user="root",
|
||||
password="Coolbot123",
|
||||
database="coolbot_data",
|
||||
charset="utf8mb4"
|
||||
)
|
||||
return cls._pool
|
||||
|
||||
@contextmanager
|
||||
def get_connection(self):
|
||||
conn = self.get_pool().get_connection()
|
||||
@contextmanager
|
||||
def get_db():
|
||||
"""Database connection context manager"""
|
||||
conn = None
|
||||
try:
|
||||
conn = psycopg2.connect(**DB_CONFIG)
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@contextmanager
|
||||
def get_cursor(self, dictionary=True):
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor(dictionary=dictionary)
|
||||
try:
|
||||
yield cursor
|
||||
conn.commit()
|
||||
except Exception as e:
|
||||
if conn:
|
||||
conn.rollback()
|
||||
raise e
|
||||
finally:
|
||||
cursor.close()
|
||||
|
||||
db = Database()
|
||||
if conn:
|
||||
conn.close()
|
||||
|
||||
def test_connection():
|
||||
"""测试数据库连接"""
|
||||
try:
|
||||
with db.get_cursor() as cursor:
|
||||
cursor.execute("SELECT 1 as test")
|
||||
result = cursor.fetchone()
|
||||
print(f"✅ 数据库连接成功: {result}")
|
||||
with get_db() as conn:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT version()")
|
||||
print(f"DB OK: {str(cur.fetchone()[0])[:50]}")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"❌ 数据库连接失败: {e}")
|
||||
print(f"DB ERROR: {e}")
|
||||
return False
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_connection()
|
||||
|
|
|
|||
Loading…
Reference in New Issue