import os import time import requests from crawv2 import get_movie_ophim, init_db from datetime import datetime, timedelta import json import re from concurrent.futures import ThreadPoolExecutor, TimeoutError def get_log_file_path(): # Log folder will be in the same directory as the script script_dir = os.path.dirname(os.path.abspath(__file__)) log_dir = os.path.join(script_dir, "log") os.makedirs(log_dir, exist_ok=True) date_str = datetime.now().strftime("%Y-%m-%d") log_file = os.path.join(log_dir, f"ophim-crawl_error_{date_str}.log") # Check file size and append a counter if larger than 5MB (5 * 1024 * 1024 bytes) max_size = 5 * 1024 * 1024 counter = 1 base_log_file = log_file while os.path.exists(log_file) and os.path.getsize(log_file) > max_size: log_file = os.path.join(log_dir, f"ophim-crawl_error_{date_str}_{counter}.log") counter += 1 return log_file def log_error(movie_name, slug, error_msg): log_file = get_log_file_path() timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") log_entry = f"[{timestamp}] Phim: {movie_name} (slug: {slug}) - Lỗi: {error_msg}\n" with open(log_file, "a", encoding="utf-8") as f: f.write(log_entry) def fetch_movie_list(page, api_url="https://ophim1.com/danh-sach/phim-moi-cap-nhat?page={}"): try: response = requests.get(api_url.format(page), timeout=10) # 10-second timeout for API call response.raise_for_status() data = response.json() if not data.get("status", False): print("API không trả về dữ liệu hợp lệ") return [], 0 return data.get("items", []), data.get("pagination", {}).get("totalPages", 1) except requests.Timeout: print(f"Hết thời gian khi gọi API danh sách phim trang {page}") return [], 0 except requests.RequestException as e: print(f"Lỗi khi gọi API danh sách phim: {str(e)}") return [], 0 def fetch_movie_details(slug): try: api_url = f"https://ophim1.com/phim/{slug}" response = requests.get(api_url, timeout=10) # 10-second timeout for API call response.raise_for_status() data = response.json() if not data.get("status", False): return None return data.get("movie", {}) except requests.Timeout: print(f"Hết thời gian khi gọi API chi tiết phim {slug}") return None except requests.RequestException as e: print(f"Lỗi khi gọi API chi tiết phim {slug}: {str(e)}") return None def determine_movie_path(base_path, movie): movie_type = movie.get("type", "single") country_slug = movie.get("country", [{}])[0].get("slug", "") if movie.get("country") else "" categories = movie.get("category", []) # Check for "phim-18" category has_phim_18 = any(cat.get("slug", "") == "phim-18" for cat in categories) if has_phim_18: return os.path.join(base_path, "Phim 18") # Country slug to folder name mapping country_folder_map = { "viet-nam": "Việt Nam", "trung-quoc": "Trung Quốc", "thai-lan": "Thái Lan", "hong-kong": "Hồng Kông", "dai-loan": "Đài Loan", "an-do": "Ấn Độ", "han-quoc": "Hàn Quốc" } # Base folder determination if movie_type == "hoathinh": base_folder = "Hoạt hình" # Check "time" field for "/" to determine if it's a series or single time_str = movie.get("time", "") is_series = "/" in time_str # Presence of "/" indicates a series (e.g., "23 phút/tập") sub_folder = "Phim bộ" if is_series else "Phim lẻ" return os.path.join(base_path, base_folder, sub_folder) elif movie_type == "tvshows": return os.path.join(base_path, "TvShows") elif movie_type == "single": return os.path.join(base_path, "Phim lẻ") elif movie_type == "series": base_folder = "Phim bộ" if country_slug in country_folder_map: sub_folder = country_folder_map[country_slug] elif country_slug in ["au-my", "anh"]: sub_folder = "Âu Mỹ" else: sub_folder = "Quốc Gia Khác" return os.path.join(base_path, base_folder, sub_folder) return base_path def process_movie(slug, movie_name, movie_path, path_db): start_time = time.time() result = get_movie_ophim( slug=slug, path=movie_path, skipad=True, replace=False, create_strm_file=True, path_db=path_db ) duration = time.time() - start_time return result, duration def crawl_movies(base_path, path_db, max_pages=1): # Initialize database init_db(path_db) # Ensure base path exists os.makedirs(base_path, exist_ok=True) current_page = 1 total_pages = 1 while current_page <= max_pages and current_page <= total_pages: print(f"Đang cào trang {current_page}/{min(max_pages, total_pages)}...") movies, total_pages = fetch_movie_list(current_page) for movie in movies: slug = movie.get("slug", "") movie_name = movie.get("name", slug) if not slug: print("Không tìm thấy slug cho phim, bỏ qua...") log_error(movie_name, slug, "Không tìm thấy slug") continue # Fetch movie details for accurate folder determination movie_details = fetch_movie_details(slug) if not movie_details: print(f"Không lấy được chi tiết phim {movie_name}, bỏ qua...") log_error(movie_name, slug, "Không lấy được chi tiết phim từ API") continue # Determine path using data from fetch_movie_details movie_path = determine_movie_path(base_path, movie_details) os.makedirs(movie_path, exist_ok=True) # Process movie with a timeout of 180 seconds try: with ThreadPoolExecutor(max_workers=1) as executor: future = executor.submit(process_movie, slug, movie_name, movie_path, path_db) result, duration = future.result(timeout=600) # 180-second timeout per movie if result["status"]: print(f"Xử lý thành công phim: {movie_name} tại {movie_path} (thời gian: {duration:.2f} giây)") else: print(f"Lỗi khi xử lý phim {movie_name}: {result['msg']} (thời gian: {duration:.2f} giây)") log_error(movie_name, slug, result["msg"]) except TimeoutError: print(f"Hết thời gian xử lý phim {movie_name} sau 180 giây, bỏ qua...") log_error(movie_name, slug, "Hết thời gian xử lý sau 180 giây") continue except Exception as e: print(f"Lỗi bất ngờ khi xử lý phim {movie_name}: {str(e)}") log_error(movie_name, slug, f"Lỗi bất ngờ: {str(e)}") continue current_page += 1 print(f"Hoàn tất cào {current_page - 1} trang tại {datetime.now()}") def main(): base_path = "D:\\Webphim\\Ophim" path_db = "D:\\Webphim\\config\\data-ophim.db" max_pages = 10 # Số trang tối đa muốn cào, có thể điều chỉnh while True: print(f"Bắt đầu cào phim lúc {datetime.now()}") crawl_movies(base_path, path_db, max_pages) print("Đang chờ 12 giờ để cào lại...") time.sleep(12 * 60 * 60) # Chờ 12 giờ if __name__ == "__main__": main()