diff --git a/crawlers/crawl_today.py b/crawlers/crawl_today.py index 8f41231..e95cebc 100644 --- a/crawlers/crawl_today.py +++ b/crawlers/crawl_today.py @@ -310,65 +310,80 @@ class YichensTodaySpider(PaginationSpider): if not post or not post.get("post_id"): return False - sql = """ - INSERT INTO yichens_posts ( - post_id, title, content, category, post_type, - price, price_unit, special_types, number_features, - author_username, author_id, contact, - has_lifebuoy, reply_count, view_count, - post_time, crawled_at, updated_at, url - ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW(), NOW(), %s) - ON CONFLICT (post_id) DO UPDATE SET - title = EXCLUDED.title, - content = EXCLUDED.content, - category = EXCLUDED.category, - post_type = EXCLUDED.post_type, - price = EXCLUDED.price, - price_unit = EXCLUDED.price_unit, - special_types = EXCLUDED.special_types, - author_username = EXCLUDED.author_username, - contact = EXCLUDED.contact, - has_lifebuoy = EXCLUDED.has_lifebuoy, - post_time = EXCLUDED.post_time, - updated_at = NOW(), - crawled_at = NOW(), - url = EXCLUDED.url - """ + import psycopg2.sql as sql has_lifebuoy = post.get("special_types") and "救生圈" in post.get("special_types", "") + cols = ['post_id', 'title', 'content', 'category', 'post_type', 'price', + 'price_unit', 'special_types', 'number_features', 'author_username', + 'author_id', 'contact', 'has_lifebuoy', 'reply_count', 'view_count', + 'post_time', 'crawled_at', 'updated_at', 'url', 'instance_id'] + + vals = [ + sql.Literal(post.get("post_id")), + sql.Literal(post.get("title")), + sql.Literal(post.get("content")), + sql.Literal(post.get("category")), + sql.Literal(post.get("post_type")), + sql.Literal(post.get("price")), + sql.Literal(post.get("price_unit")), + sql.Literal(post.get("special_types")), + sql.Literal(post.get("number_features")), + sql.Literal(post.get("author_username")), + sql.Literal(f"user_{post.get('post_id')}"), + sql.Literal(post.get("contact")), + sql.Literal(has_lifebuoy), + sql.Literal(0), + sql.Literal(0), + sql.Literal(post.get("post_time")), + sql.SQL('NOW()'), + sql.SQL('NOW()'), + sql.Literal(post.get("url")), + sql.Literal(self.instance_id), + ] + + update_assigns = [ + "title = EXCLUDED.title", + "content = EXCLUDED.content", + "category = EXCLUDED.category", + "post_type = EXCLUDED.post_type", + "price = EXCLUDED.price", + "price_unit = EXCLUDED.price_unit", + "special_types = EXCLUDED.special_types", + "number_features = EXCLUDED.number_features", + "author_username = EXCLUDED.author_username", + "author_id = EXCLUDED.author_id", + "contact = EXCLUDED.contact", + "has_lifebuoy = EXCLUDED.has_lifebuoy", + "reply_count = EXCLUDED.reply_count", + "view_count = EXCLUDED.view_count", + "post_time = EXCLUDED.post_time", + "updated_at = NOW()", + "crawled_at = NOW()", + "url = EXCLUDED.url", + "instance_id = EXCLUDED.instance_id", + ] + + query = sql.SQL("INSERT INTO yichens_posts ({}) VALUES ({}) ON CONFLICT (post_id) DO UPDATE SET {}").format( + sql.SQL(', ').join(sql.Identifier(c) for c in cols), + sql.SQL(', ').join(vals), + sql.SQL(', ').join(sql.SQL(a) for a in update_assigns), + ) + try: with get_db() as conn: if not conn: print("数据库连接失败") return False cur = conn.cursor() - cur.execute(sql, ( - post.get("post_id"), - post.get("title"), - post.get("content"), - post.get("category"), - post.get("post_type"), - post.get("price"), - post.get("price_unit"), - post.get("special_types"), - post.get("number_features"), - post.get("author_username"), - f"user_{post.get('post_id')}", - post.get("contact"), - has_lifebuoy, - 0, - 0, - post.get("post_time"), - post.get("url"), - )) + cur.execute(query) conn.commit() cur.close() return True except Exception as e: print(f"保存失败: {e}") return False - + def run(self): log_id = self._log_start() try: