feat(scripts): 添加 YouTube Studio 群组名到 entity_id 批量解析脚本
支持 Playwright 自动捕获鉴权、requests 回放查询、Excel/CSV 输出，包含在线捕获和离线回放两种模式
This commit is contained in:
781
scripts/lookup_groups.py
Normal file
781
scripts/lookup_groups.py
Normal file
@@ -0,0 +1,781 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
r"""
|
||||
YouTube Studio 内容管理器「群组名 -> entity_id（groupId）」批量解析脚本
|
||||
（Playwright 自动捕获鉴权套件 + requests 回放查询 + Excel/CSV 输出）。
|
||||
|
||||
## 已实证的查询机制（与 yt-studio-group-id-lookup skill 一致）
|
||||
|
||||
1. 页面自身的内部接口：
|
||||
POST https://studio.youtube.com/youtubei/v1/yta_web/search_groups?alt=json
|
||||
请求体 `query` 字段为搜索词，响应形如：
|
||||
{ "groupDatas": [ { "displayName": "...", "groupId": "..." } ] }
|
||||
`groupId` 即 explore URL 里的 `entity_id`。
|
||||
|
||||
2. 手写请求必 401：鉴权依赖页面 JS 逐请求计算的
|
||||
`Authorization: SAPISIDHASH...`、锁定查询所有者的 `X-YouTube-Delegation-Context`，
|
||||
以及浏览器自动携带的 Cookie。唯一可靠路径是
|
||||
「捕获页面自己的请求 -> 原样回放，只改 query」。
|
||||
|
||||
3. 本脚本用 Playwright 打开所有者分析页并监听 `search_groups` 响应：
|
||||
先尝试自动触发一次搜索（选择器猜不中时提示在浏览器里手动搜一次，任意词即可），
|
||||
从该请求拿到完整套件（全部请求头含 Cookie + 请求体模板），随后在 Python 侧
|
||||
并发回放全量名单。多所有者时逐个页面各捕一份套件，首个精确命中即停。
|
||||
|
||||
## 使用前提
|
||||
- 依赖：playwright、requests、openpyxl（项目环境 `uv sync` 后 `uv run python ...`）。
|
||||
- 必须复用「已登录 YouTube Studio」的浏览器会话（同 youtube_export_interceptor.py）：
|
||||
- `--user-data-dir` + `--channel chrome|msedge`：用已登录用户数据目录启动（需先关闭该浏览器）
|
||||
- `--connect http://localhost:9222`：附加到已在调试端口运行的浏览器
|
||||
（先 `chrome.exe --remote-debugging-port=9222` 或 `msedge.exe --remote-debugging-port=9222`）
|
||||
- 待查群组名单（`--names`）：json / txt / csv / xlsx 均可。
|
||||
|
||||
## 用法
|
||||
# 0) 纯逻辑自测（无需浏览器/网络/第三方库）
|
||||
python lookup_groups.py --selftest
|
||||
|
||||
# 1) 在线模式：打开所有者页面 -> 捕获套件 -> 回放全量名单 -> 输出 Excel
|
||||
python lookup_groups.py --url "<owner analytics URL>" --names 名单.json ^
|
||||
--channel chrome --user-data-dir "%LOCALAPPDATA%\Google\Chrome\User Data"
|
||||
python lookup_groups.py --url "<owner1 URL>" --url "<owner2 URL>" --names 名单.xlsx ^
|
||||
--connect http://localhost:9222 --save-bundles bundles.json
|
||||
|
||||
# 2) 离线回放：套件已存盘（--save-bundles 产物），无需浏览器
|
||||
python lookup_groups.py --bundles bundles.json --names 名单.json --out result.xlsx
|
||||
|
||||
套件 bundles.json（每个所有者一个对象，`--save-bundles` 自动生成，也可手工维护）：
|
||||
{ "ownerId": "...", "ownerDisplay": "...", "url": "...",
|
||||
"headers": { "Authorization": "SAPISIDHASH ...", "Cookie": "...",
|
||||
"X-YouTube-Delegation-Context": "...", "...": "..." },
|
||||
"bodyTemplate": { "...": "...", "query": "" } }
|
||||
|
||||
名单 names 支持的格式：
|
||||
json : ["名字1", "名字2"] 或 [["GROUP_NAME"], ["名字1"]] 或 {"names": [...]}
|
||||
txt : 每行一个名字
|
||||
csv/xlsx: 自动识别 群组名称 / 群组 / group_name / GROUP_NAME / 实体名称 / 名称 列
|
||||
|
||||
输出（默认 group_entity_id_result.xlsx，缺 openpyxl 自动回退 csv）：
|
||||
group_name / ownerid / owner_display / groupid / 备注
|
||||
多所有者按顺序逐个尝试，首个精确命中即停；全部未命中把候选写进备注便于人工复核。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
ENDPOINT = "https://studio.youtube.com/youtubei/v1/yta_web/search_groups?alt=json"
|
||||
SEARCH_GROUPS_PATH = "/youtubei/v1/yta_web/search_groups"
|
||||
|
||||
OWNER_PATH_RE = re.compile(r"studio\.youtube\.com/owner/([A-Za-z0-9_-]+)")
|
||||
OWNER_QUERY_RE = re.compile(r"[?&]o=([A-Za-z0-9_-]+)")
|
||||
|
||||
# 回放时剥离的请求头：长度/传输类由 requests 自行计算，accept-encoding 防止收到解不开的 br。
|
||||
STRIP_HEADERS = {
|
||||
"content-length", "host", "connection", "keep-alive",
|
||||
"transfer-encoding", "accept-encoding", "content-encoding",
|
||||
}
|
||||
|
||||
HTTP_HINTS = {
|
||||
401: "（鉴权失败：套件缺 Authorization/Cookie 或已过期，请重新捕获）",
|
||||
403: "（无权限或 delegation 语境不符）",
|
||||
429: "（限流：调低 --max-workers 或稍后重试）",
|
||||
}
|
||||
|
||||
REQUEST_TIMEOUT = 30
|
||||
MANUAL_WAIT_SECONDS = 180
|
||||
|
||||
RESULT_HEADER = ["group_name", "ownerid", "owner_display", "groupid", "备注"]
|
||||
|
||||
# 名单列名别名（规范化为 strip+lower 后比较）
|
||||
NAME_COLUMN_ALIASES = {
|
||||
"group_name", "groupname", "群组名称", "群组", "group", "name", "名称",
|
||||
"实体名称", "group name",
|
||||
}
|
||||
|
||||
USAGE_HINT = (
|
||||
"\n实际运行需先 uv sync（或 pip install playwright requests openpyxl），"
|
||||
"并复用已登录 YouTube Studio 的浏览器会话（二选一）：\n"
|
||||
' 方式 A: python lookup_groups.py --url "<owner analytics URL>" --names 名单.json '
|
||||
'--channel chrome --user-data-dir "%LOCALAPPDATA%\\Google\\Chrome\\User Data"\n'
|
||||
" 方式 B: 先 chrome.exe --remote-debugging-port=9222，再\n"
|
||||
' python lookup_groups.py --url "<owner analytics URL>" --names 名单.json '
|
||||
"--connect http://localhost:9222\n"
|
||||
" 离线回放（套件已存盘）: python lookup_groups.py --bundles bundles.json --names 名单.json\n"
|
||||
"多所有者: --url 可重复多次，每个所有者页面各触发一次搜索即可。"
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 纯逻辑：URL / 名单解析 #
|
||||
# --------------------------------------------------------------------------- #
|
||||
def parse_owner_id(url: str) -> str:
|
||||
"""从 owner 分析页 URL 提取 ownerId（路径 /owner/<id>/ 优先，其次 ?o=<id>）。"""
|
||||
m = OWNER_PATH_RE.search(url or "")
|
||||
if m:
|
||||
return m.group(1)
|
||||
m = OWNER_QUERY_RE.search(url or "")
|
||||
if m:
|
||||
return m.group(1)
|
||||
return ""
|
||||
|
||||
|
||||
def _norm_key(s) -> str:
|
||||
return str(s).strip().lower()
|
||||
|
||||
|
||||
def parse_names_payload(data) -> list:
|
||||
"""json 结构 -> 名字列表（纯列表 / 带表头二维列表 / {"names": [...]}），保序去重。"""
|
||||
if isinstance(data, dict):
|
||||
for key in ("names", "group_names", "groups", "group_name"):
|
||||
if isinstance(data.get(key), list):
|
||||
data = data[key]
|
||||
break
|
||||
else:
|
||||
return []
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
names: list = []
|
||||
seen = set()
|
||||
for item in data:
|
||||
if isinstance(item, (list, tuple)):
|
||||
if not item:
|
||||
continue
|
||||
value = str(item[0]).strip()
|
||||
if _norm_key(value) in NAME_COLUMN_ALIASES: # 表头行
|
||||
continue
|
||||
else:
|
||||
value = str(item).strip()
|
||||
if value and value not in seen:
|
||||
seen.add(value)
|
||||
names.append(value)
|
||||
return names
|
||||
|
||||
|
||||
def _names_from_rows(rows: list) -> list:
|
||||
"""二维行（含表头行）-> 名字列表；自动找别名列，单列无表头全当名字。"""
|
||||
if not rows:
|
||||
return []
|
||||
first = rows[0]
|
||||
alias_col = None
|
||||
for idx, cell in enumerate(first):
|
||||
if _norm_key(cell) in NAME_COLUMN_ALIASES:
|
||||
alias_col = idx
|
||||
break
|
||||
if alias_col is not None:
|
||||
data_rows = rows[1:]
|
||||
|
||||
def pick(r):
|
||||
return r[alias_col] if alias_col < len(r) else ""
|
||||
elif len(first) == 1:
|
||||
data_rows = rows # 单列且首行不是表头 -> 全部当名字
|
||||
|
||||
def pick(r):
|
||||
return r[0] if r else ""
|
||||
else:
|
||||
raise ValueError(
|
||||
f"无法识别名字列，第一行: {first}（表头用 群组名称/group_name，或改为单列文件）")
|
||||
|
||||
names, seen = [], set()
|
||||
for r in data_rows:
|
||||
v = str(pick(r)).strip()
|
||||
if v and _norm_key(v) not in ("nan", "none") and v not in seen:
|
||||
seen.add(v)
|
||||
names.append(v)
|
||||
return names
|
||||
|
||||
|
||||
def load_names(path: str) -> list:
|
||||
"""按扩展名加载名单：json / txt / csv / xlsx。"""
|
||||
ext = os.path.splitext(path)[1].lower()
|
||||
if ext == ".json":
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return parse_names_payload(json.load(f))
|
||||
if ext in (".xlsx", ".xls"):
|
||||
import pandas as pd
|
||||
|
||||
df = pd.read_excel(path, dtype=str)
|
||||
rows = [[str(c) for c in df.columns]]
|
||||
rows += [["" if v is None else str(v) for v in rec]
|
||||
for rec in df.itertuples(index=False)]
|
||||
return _names_from_rows(rows)
|
||||
# txt / csv 统一按 csv 解析（txt 每行一个名字天然兼容）
|
||||
with open(path, newline="", encoding="utf-8-sig") as f:
|
||||
rows = [r for r in csv.reader(f) if any(c.strip() for c in r)]
|
||||
return _names_from_rows(rows)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 纯逻辑：请求构造 / 匹配 / 回放 #
|
||||
# --------------------------------------------------------------------------- #
|
||||
def build_body(body_template: dict, query: str) -> dict:
|
||||
"""深拷贝模板并只改 query（其余字段一律不动，否则查错所有者或 401）。"""
|
||||
body = json.loads(json.dumps(body_template or {}))
|
||||
body["query"] = query
|
||||
return body
|
||||
|
||||
|
||||
def pick_match(group_datas: list, name: str):
|
||||
"""精确 displayName 命中 -> (groupId, [])；否则 (None, 前 5 个候选 '名=ID')。"""
|
||||
for g in group_datas:
|
||||
if g.get("displayName") == name:
|
||||
return g.get("groupId"), []
|
||||
cands = [f"{g.get('displayName')}={g.get('groupId')}" for g in group_datas[:5]]
|
||||
return None, cands
|
||||
|
||||
|
||||
def _external_owner_id(body: dict) -> str:
|
||||
"""取请求体 context.user.delegationContext.externalOwnerId（锁定查询所有者）。"""
|
||||
try:
|
||||
return body["context"]["user"]["delegationContext"]["externalOwnerId"]
|
||||
except (KeyError, TypeError):
|
||||
return ""
|
||||
|
||||
|
||||
_TLS = threading.local()
|
||||
|
||||
|
||||
def _thread_session():
|
||||
"""线程本地 requests.Session（延迟导入：selftest/单测无需安装 requests）。"""
|
||||
import requests
|
||||
|
||||
s = getattr(_TLS, "session", None)
|
||||
if s is None:
|
||||
s = _TLS.session = requests.Session()
|
||||
return s
|
||||
|
||||
|
||||
def search(bundle: dict, name: str, session=None) -> dict:
|
||||
"""对单个所有者套件回放一次 search_groups，只改 query。
|
||||
|
||||
session 可注入测试替身（.post(url, json=..., headers=..., timeout=...)）。
|
||||
"""
|
||||
headers = {k: v for k, v in (bundle.get("headers") or {}).items()
|
||||
if str(k).lower() not in STRIP_HEADERS}
|
||||
body = build_body(bundle.get("bodyTemplate") or {}, name)
|
||||
url = bundle.get("url") or ENDPOINT
|
||||
try:
|
||||
if session is None:
|
||||
session = _thread_session()
|
||||
r = session.post(url, json=body, headers=headers, timeout=REQUEST_TIMEOUT)
|
||||
except Exception as e: # noqa: BLE001
|
||||
return {"status": "ERR", "detail": str(e)}
|
||||
|
||||
code = getattr(r, "status_code", 0)
|
||||
if code != 200:
|
||||
return {"status": "HTTP", "detail": code, "hint": HTTP_HINTS.get(code, "")}
|
||||
try:
|
||||
data = r.json()
|
||||
except Exception as e: # noqa: BLE001
|
||||
return {"status": "ERR", "detail": f"响应非 JSON: {e}"}
|
||||
|
||||
group_id, cands = pick_match(data.get("groupDatas") or [], name)
|
||||
if group_id:
|
||||
return {"status": "OK", "groupId": group_id}
|
||||
return {"status": "NF", "cands": cands}
|
||||
|
||||
|
||||
def resolve(name: str, bundles: list, session=None) -> list:
|
||||
"""多所有者逐个尝试，首个精确命中即停并记录 ownerid，未命中收集各所有者线索。"""
|
||||
notes = []
|
||||
for b in bundles:
|
||||
if not isinstance(b, dict):
|
||||
continue
|
||||
owner_id = b.get("ownerId", "")
|
||||
owner = b.get("ownerDisplay", "")
|
||||
r = search(b, name, session)
|
||||
if r["status"] == "OK":
|
||||
note = "精确命中" if not notes else "精确命中；此前 " + "; ".join(notes)
|
||||
return [name, owner_id, owner, r["groupId"], note]
|
||||
if r["status"] == "HTTP":
|
||||
msg = f"{owner}({owner_id}) 返回 HTTP{r['detail']}{r.get('hint', '')}"
|
||||
notes.append(msg)
|
||||
elif r["status"] == "ERR":
|
||||
notes.append(f"{owner}({owner_id}) 网络错: {r['detail'][:60]}")
|
||||
elif r["status"] == "NF":
|
||||
cands = r.get("cands") or []
|
||||
notes.append(
|
||||
f"{owner}({owner_id}) 候选: {', '.join(cands)}" if cands
|
||||
else f"{owner}({owner_id}) 无结果")
|
||||
return [name, "", "", "", "未匹配任何 owner；" + " | ".join(notes)]
|
||||
|
||||
|
||||
def validate_bundle(bundle: dict) -> list:
|
||||
"""校验套件完整性，返回问题列表（鉴权头缺失是 401 的头号根因，提前点破）。"""
|
||||
problems = []
|
||||
headers = {_norm_key(k): v for k, v in (bundle.get("headers") or {}).items()}
|
||||
if not headers:
|
||||
problems.append("headers 为空，回放必 401")
|
||||
else:
|
||||
for key, desc in (
|
||||
("authorization", "Authorization（SAPISIDHASH）"),
|
||||
("cookie", "Cookie"),
|
||||
("x-youtube-delegation-context", "X-YouTube-Delegation-Context"),
|
||||
):
|
||||
if not headers.get(key):
|
||||
problems.append(f"缺少 {desc} 头，回放大概率 401 或查错所有者")
|
||||
body = bundle.get("bodyTemplate") or {}
|
||||
if not body:
|
||||
problems.append("bodyTemplate 为空")
|
||||
elif not _external_owner_id(body):
|
||||
problems.append("bodyTemplate 缺 context.user.delegationContext，可能查错所有者")
|
||||
return problems
|
||||
|
||||
|
||||
def merge_bundles(existing: list, captured: list) -> list:
|
||||
"""合并套件：captured 按 ownerId 覆盖同名项，其余按原序保留。"""
|
||||
by_owner: dict = {}
|
||||
for b in existing:
|
||||
if isinstance(b, dict) and b.get("ownerId"):
|
||||
by_owner[b["ownerId"]] = b
|
||||
for b in captured:
|
||||
if isinstance(b, dict) and b.get("ownerId"):
|
||||
by_owner[b["ownerId"]] = b
|
||||
return list(by_owner.values())
|
||||
|
||||
|
||||
def write_out(path: str, header: list, rows: list) -> str:
|
||||
"""写结果文件：.xlsx 优先 openpyxl，缺库自动回退 .csv。返回实际写入路径。"""
|
||||
if path.lower().endswith(".xlsx"):
|
||||
try:
|
||||
from openpyxl import Workbook
|
||||
except ImportError:
|
||||
print("[i] 缺少 openpyxl，回退为 csv 输出")
|
||||
path = path[:-5] + ".csv"
|
||||
else:
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws.title = "result"
|
||||
ws.append(header)
|
||||
for r in rows:
|
||||
ws.append(r)
|
||||
for col, w in zip("ABCDE", (28, 22, 24, 22, 60)):
|
||||
ws.column_dimensions[col].width = w
|
||||
wb.save(path)
|
||||
return path
|
||||
|
||||
with open(path, "w", newline="", encoding="utf-8-sig") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(header)
|
||||
w.writerows(rows)
|
||||
return path
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 浏览器侧：自动捕获套件 #
|
||||
# --------------------------------------------------------------------------- #
|
||||
AUTO_SEARCH_SELECTORS = (
|
||||
"input[placeholder*='搜索']",
|
||||
"input[aria-label*='搜索']",
|
||||
"input[placeholder*='Search']",
|
||||
"input[aria-label*='Search']",
|
||||
"ytcp-analytics-filter-bar input",
|
||||
"ytcp-text-input input",
|
||||
)
|
||||
|
||||
OWNER_DISPLAY_SELECTORS = (
|
||||
"ytcp-account-item button",
|
||||
".account-switcher button",
|
||||
"#owner-name",
|
||||
)
|
||||
|
||||
|
||||
def _launch_page(p, user_data_dir, channel, cdp_url):
|
||||
"""按 interceptor 同款三种方式拿到 (browser, context, page)。"""
|
||||
if cdp_url:
|
||||
browser = p.chromium.connect_over_cdp(cdp_url)
|
||||
context = browser.contexts[0] if browser.contexts else browser.new_context()
|
||||
return browser, context, context.new_page()
|
||||
if user_data_dir:
|
||||
context = p.chromium.launch_persistent_context(
|
||||
user_data_dir=user_data_dir,
|
||||
channel=channel,
|
||||
headless=False,
|
||||
args=["--disable-blink-features=AutomationControlled"],
|
||||
)
|
||||
return None, context, context.new_page()
|
||||
browser = p.chromium.launch(headless=False, channel=channel)
|
||||
context = browser.new_context()
|
||||
return browser, context, context.new_page()
|
||||
|
||||
|
||||
def _try_auto_search(page, probe="a"):
|
||||
"""尽力自动触发一次搜索（选择器猜不中就交回手动流程，结果无关紧要）。"""
|
||||
for sel in AUTO_SEARCH_SELECTORS:
|
||||
try:
|
||||
loc = page.locator(sel).first
|
||||
loc.wait_for(state="visible", timeout=1500)
|
||||
loc.fill(probe)
|
||||
try:
|
||||
loc.press("Enter")
|
||||
except Exception:
|
||||
pass
|
||||
page.wait_for_timeout(1500)
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
return False
|
||||
|
||||
|
||||
def _read_owner_display(page):
|
||||
"""尽力读所有者/账号显示名（仅用于结果标注，读不到不影响功能）。"""
|
||||
for sel in OWNER_DISPLAY_SELECTORS:
|
||||
try:
|
||||
text = page.locator(sel).first.inner_text(timeout=2000).strip()
|
||||
if text:
|
||||
return text
|
||||
except Exception:
|
||||
continue
|
||||
return ""
|
||||
|
||||
|
||||
def _capture_bundle(page, url, wait_seconds, owner_display=""):
|
||||
"""打开一个所有者分析页，捕获一次 search_groups 请求 -> 完整套件。"""
|
||||
owner_id = parse_owner_id(url)
|
||||
print(f"[捕获] 打开所有者页面: {owner_display or owner_id or url}")
|
||||
try:
|
||||
page.goto(url, wait_until="domcontentloaded")
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f"[捕获] 页面加载较慢（{type(e).__name__}），继续等待搜索……", file=sys.stderr)
|
||||
|
||||
if "accounts.google" in page.url:
|
||||
raise SystemExit(
|
||||
"[!] 当前会话未登录（跳转到 Google 登录页）。请用 --user-data-dir 或 "
|
||||
"--connect 复用已登录浏览器后重试。")
|
||||
|
||||
captured = []
|
||||
|
||||
def on_response(resp):
|
||||
try:
|
||||
if SEARCH_GROUPS_PATH in resp.url:
|
||||
captured.append(resp)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
page.on("response", on_response)
|
||||
try:
|
||||
if _try_auto_search(page):
|
||||
print("[捕获] 已尝试自动触发一次群组搜索……")
|
||||
# 自动触发后先安静等 5 秒；仍无请求再提示手动搜索
|
||||
quiet = time.time() + 5
|
||||
while not captured and time.time() < quiet:
|
||||
page.wait_for_timeout(500)
|
||||
deadline = time.time() + wait_seconds
|
||||
notified = False
|
||||
last_report = time.time()
|
||||
while not captured and time.time() < deadline:
|
||||
if not notified:
|
||||
print(f"[捕获] 请在打开的浏览器窗口中，于该所有者分析页顶部的搜索/筛选框"
|
||||
f"输入任意词并回车（只需触发一次搜索，结果无所谓）。"
|
||||
f"最长等待 {wait_seconds} 秒……")
|
||||
notified = True
|
||||
page.wait_for_timeout(1000)
|
||||
if time.time() - last_report >= 30:
|
||||
print(f"[捕获] 仍在等待手动搜索……剩余 {int(deadline - time.time())} 秒")
|
||||
last_report = time.time()
|
||||
if not captured:
|
||||
raise TimeoutError(
|
||||
f"{wait_seconds} 秒内未捕获到 search_groups 请求。"
|
||||
"请确认页面是高级模式分析页且搜索框可用；若页面被重定向到登录页，"
|
||||
"请改用 --user-data-dir / --connect 复用已登录会话后重试。")
|
||||
finally:
|
||||
try:
|
||||
page.remove_listener("response", on_response)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
resp = captured[-1] # 取最近一次（SAPISIDHASH 时间戳最新）
|
||||
req = resp.request
|
||||
try:
|
||||
headers = dict(req.all_headers())
|
||||
except Exception: # noqa: BLE001
|
||||
headers = dict(req.headers)
|
||||
try:
|
||||
body = json.loads(req.post_data or "{}")
|
||||
except Exception: # noqa: BLE001
|
||||
body = {}
|
||||
|
||||
ext_owner = _external_owner_id(body)
|
||||
if ext_owner and owner_id and ext_owner != owner_id:
|
||||
print(f"[捕获] 警告：请求体 externalOwnerId={ext_owner} 与 URL ownerId={owner_id} "
|
||||
"不一致，以请求体为准。", file=sys.stderr)
|
||||
owner_id = ext_owner
|
||||
owner_id = owner_id or ext_owner
|
||||
if not owner_id:
|
||||
raise ValueError("无法确定 ownerId（URL 非 /owner/<id>/ 且请求体无 delegationContext）")
|
||||
|
||||
bundle = {
|
||||
"ownerId": owner_id,
|
||||
"ownerDisplay": owner_display or _read_owner_display(page),
|
||||
"url": req.url,
|
||||
"headers": headers,
|
||||
"bodyTemplate": body,
|
||||
"capturedAt": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
}
|
||||
for pb in validate_bundle(bundle):
|
||||
print(f"[捕获] 警告：{pb}", file=sys.stderr)
|
||||
print(f"[捕获] 套件就绪：owner={bundle['ownerDisplay'] or owner_id} ({owner_id})，"
|
||||
f"headers={len(headers)} 项，请求体模板已取得")
|
||||
return bundle
|
||||
|
||||
|
||||
def capture_bundles(urls, user_data_dir=None, channel=None, cdp_url=None,
|
||||
owner_display="", wait_seconds=MANUAL_WAIT_SECONDS):
|
||||
"""打开浏览器，逐个所有者捕获套件（同一个浏览器会话依次 goto）。"""
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
except ImportError:
|
||||
raise SystemExit("[!] 缺少 playwright：uv sync 或 pip install playwright 后重试。")
|
||||
|
||||
mode = "CDP 附加" if cdp_url else ("用户数据目录" if user_data_dir else "全新会话（大概率未登录）")
|
||||
print(f"[捕获] 浏览器会话：{mode}")
|
||||
|
||||
bundles = []
|
||||
with sync_playwright() as p:
|
||||
try:
|
||||
browser, context, page = _launch_page(p, user_data_dir, channel, cdp_url)
|
||||
except SystemExit:
|
||||
raise
|
||||
except Exception as e: # noqa: BLE001
|
||||
raise SystemExit(
|
||||
f"[!] 启动/连接浏览器失败: {e}\n"
|
||||
" 方式 A 需先关闭对应浏览器；方式 B 先以调试端口启动：\n"
|
||||
" chrome.exe --remote-debugging-port=9222")
|
||||
try:
|
||||
for i, url in enumerate(urls, 1):
|
||||
print(f"[捕获] 所有者 {i}/{len(urls)}")
|
||||
display = owner_display if len(urls) == 1 else ""
|
||||
bundles.append(_capture_bundle(page, url, wait_seconds, display))
|
||||
finally:
|
||||
if browser is not None:
|
||||
browser.close()
|
||||
elif context is not None:
|
||||
context.close()
|
||||
return bundles
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 自测（无需浏览器/网络/第三方库，仅标准库） #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class _FakeResp:
|
||||
def __init__(self, code, payload=None):
|
||||
self.status_code = code
|
||||
self._payload = payload
|
||||
|
||||
def json(self):
|
||||
if self._payload is None:
|
||||
raise ValueError("no json")
|
||||
return self._payload
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""按 `模板marker::query` 路由响应的测试替身，可区分不同所有者的套件。"""
|
||||
|
||||
def __init__(self, routing):
|
||||
self.routing = routing
|
||||
self.calls = []
|
||||
|
||||
def post(self, url, json=None, headers=None, timeout=None):
|
||||
self.calls.append((url, json, headers))
|
||||
body = json or {}
|
||||
key = f"{body.get('marker', '')}::{body.get('query', '')}"
|
||||
item = self.routing.get(key)
|
||||
if isinstance(item, Exception):
|
||||
raise item
|
||||
if item is not None:
|
||||
return item
|
||||
return _FakeResp(200, {"groupDatas": []})
|
||||
|
||||
|
||||
def selftest():
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
# 1) ownerId 解析（路径优先，其次 ?o=）
|
||||
url = "https://studio.youtube.com/owner/bqSUnNpU67xJ51TxH4PKpQ/analytics?o=bqSUnNpU67xJ51TxH4PKpQ"
|
||||
assert parse_owner_id(url) == "bqSUnNpU67xJ51TxH4PKpQ"
|
||||
assert parse_owner_id("https://studio.youtube.com/analytics?o=AbCdEf123") == "AbCdEf123"
|
||||
assert parse_owner_id("https://studio.youtube.com/") == ""
|
||||
|
||||
# 2) 名单解析：json 两种格式 + dict + txt + csv（表头/单列/多列别名）
|
||||
assert parse_names_payload(["A", "B", "A "]) == ["A", "B"]
|
||||
assert parse_names_payload([["GROUP_NAME"], ["A"], ["B"]]) == ["A", "B"]
|
||||
assert parse_names_payload([["群组名称", "备注"], ["A", "x"], ["B", "y"]]) == ["A", "B"]
|
||||
assert parse_names_payload({"names": ["A", "B"]}) == ["A", "B"]
|
||||
p_txt = Path(td) / "names.txt"
|
||||
p_txt.write_text("A\n B \n\nA\n", encoding="utf-8")
|
||||
assert load_names(str(p_txt)) == ["A", "B"]
|
||||
p_csv = Path(td) / "names.csv"
|
||||
p_csv.write_text("群组名称,备注\nA,x\nB,\n", encoding="utf-8-sig")
|
||||
assert load_names(str(p_csv)) == ["A", "B"]
|
||||
p_single = Path(td) / "single.csv"
|
||||
p_single.write_text("A\nB\n", encoding="utf-8")
|
||||
assert load_names(str(p_single)) == ["A", "B"]
|
||||
|
||||
# 3) 请求体构造：深拷贝不污染模板，只改 query
|
||||
tpl = {"context": {"user": {"delegationContext": {"externalOwnerId": "O1"}}},
|
||||
"query": ""}
|
||||
body = build_body(tpl, "靓舟桃")
|
||||
assert body["query"] == "靓舟桃" and tpl["query"] == ""
|
||||
|
||||
# 4) 匹配逻辑：精确命中 / 候选
|
||||
gd = [{"displayName": "X 漫剧-1", "groupId": "G1"},
|
||||
{"displayName": "X 漫剧-2", "groupId": "G2"}]
|
||||
assert pick_match(gd, "X 漫剧-2") == ("G2", [])
|
||||
gid, cands = pick_match(gd, "X 漫剧")
|
||||
assert gid is None and cands == ["X 漫剧-1=G1", "X 漫剧-2=G2"]
|
||||
|
||||
# 5) 回放：命中 / 未命中候选 / HTTP 提示 / 剥离长度类头
|
||||
# （模板里放一个 marker 字段，让替身能区分不同所有者的套件）
|
||||
bundle = {
|
||||
"ownerId": "O1", "ownerDisplay": "Owner One", "url": ENDPOINT,
|
||||
"headers": {"Authorization": "SAPISIDHASH x", "Cookie": "SID=1",
|
||||
"X-YouTube-Delegation-Context": "ctx",
|
||||
"Content-Length": "3", "Host": "studio.youtube.com"},
|
||||
"bodyTemplate": {**tpl, "marker": "O1"},
|
||||
}
|
||||
sess = _FakeSession({
|
||||
"O1::命中": _FakeResp(200, {"groupDatas": [{"displayName": "命中", "groupId": "G9"}]}),
|
||||
"O1::疑似": _FakeResp(200, {"groupDatas": gd}),
|
||||
"O1::限流": _FakeResp(429),
|
||||
})
|
||||
assert search(bundle, "命中", sess) == {"status": "OK", "groupId": "G9"}
|
||||
r = search(bundle, "疑似", sess)
|
||||
assert r["status"] == "NF" and "X 漫剧-1=G1" in r["cands"]
|
||||
r = search(bundle, "限流", sess)
|
||||
assert r["status"] == "HTTP" and "限流" in r["hint"]
|
||||
sent_headers = sess.calls[-1][2]
|
||||
assert "Content-Length" not in sent_headers and "Host" not in sent_headers
|
||||
assert sent_headers["Authorization"] == "SAPISIDHASH x"
|
||||
assert search(bundle, "断网", _FakeSession({"O1::断网": OSError("refused")}))["status"] == "ERR"
|
||||
|
||||
# 6) 多所有者归并：先 NF 后命中 / 全未命中备注
|
||||
b1 = dict(bundle, ownerId="O1", ownerDisplay="一号")
|
||||
b2 = dict(bundle, ownerId="O2", ownerDisplay="二号",
|
||||
bodyTemplate={**tpl, "marker": "O2"})
|
||||
sess2 = _FakeSession({
|
||||
"O2::跨主": _FakeResp(200, {"groupDatas": [{"displayName": "跨主", "groupId": "G2"}]}),
|
||||
})
|
||||
row = resolve("跨主", [b1, b2], sess2)
|
||||
assert row == ["跨主", "O2", "二号", "G2",
|
||||
"精确命中；此前 一号(O1) 无结果"]
|
||||
row = resolve("查无", [b1], sess2)
|
||||
assert row[3] == "" and "未匹配任何 owner" in row[4]
|
||||
|
||||
# 7) 套件校验与合并
|
||||
problems = validate_bundle({"headers": {"Authorization": "x"},
|
||||
"bodyTemplate": tpl})
|
||||
assert any("Cookie" in p for p in problems)
|
||||
merged = merge_bundles([{"ownerId": "O1", "v": 1}],
|
||||
[{"ownerId": "O1", "v": 2}, {"ownerId": "O2", "v": 3}])
|
||||
assert [b["ownerId"] for b in merged] == ["O1", "O2"] and merged[0]["v"] == 2
|
||||
|
||||
# 8) 输出：csv 路径直写（xlsx 分支见单元测试）
|
||||
out = Path(td) / "result.csv"
|
||||
path = write_out(str(out), RESULT_HEADER, [row])
|
||||
text = out.read_text(encoding="utf-8-sig")
|
||||
assert "group_name,ownerid,owner_display,groupid,备注" in text and "查无" in text
|
||||
assert path == str(out)
|
||||
|
||||
print("selftest OK：URL/名单解析、请求体、匹配、回放、多所有者、校验、输出 全部通过")
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(
|
||||
description="YouTube Studio 群组名 -> entity_id 批量解析（捕获套件 + 回放）")
|
||||
ap.add_argument("--selftest", action="store_true", help="仅跑纯逻辑自测（无需浏览器/网络）")
|
||||
ap.add_argument("--url", action="append", default=[], metavar="OWNER_URL",
|
||||
help="所有者分析页 URL，可重复多次（多所有者）")
|
||||
ap.add_argument("--names", help="群组名单：json / txt / csv / xlsx")
|
||||
ap.add_argument("--out", default="group_entity_id_result.xlsx",
|
||||
help="输出文件（.xlsx 或 .csv），默认 %(default)s")
|
||||
ap.add_argument("--bundles", help="已有套件 bundles.json（离线回放，或与 --url 捕获结果合并）")
|
||||
ap.add_argument("--save-bundles", metavar="PATH",
|
||||
help="把套件存到该 json（含 --bundles 读入的），供下次离线回放")
|
||||
ap.add_argument("--max-workers", type=int, default=8, help="并发线程数，默认 %(default)s")
|
||||
ap.add_argument("--connect", help="通过 CDP 附加到已打开浏览器，如 http://localhost:9222")
|
||||
ap.add_argument("--user-data-dir",
|
||||
help="浏览器用户数据目录（复用登录态，需先关闭该浏览器）")
|
||||
ap.add_argument("--channel", choices=["chrome", "msedge"],
|
||||
help="浏览器品牌（--user-data-dir 方式必填其一）")
|
||||
ap.add_argument("--owner-display", help="所有者显示名（可选，仅单个 --url 时用于结果标注）")
|
||||
ap.add_argument("--wait-seconds", type=int, default=MANUAL_WAIT_SECONDS,
|
||||
help="等待手动触发搜索的最长秒数，默认 %(default)s")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
if args.selftest:
|
||||
selftest()
|
||||
return
|
||||
if not args.url and not args.bundles:
|
||||
selftest()
|
||||
print(USAGE_HINT)
|
||||
return
|
||||
|
||||
bundles = []
|
||||
if args.bundles:
|
||||
try:
|
||||
with open(args.bundles, encoding="utf-8") as f:
|
||||
file_bundles = json.load(f)
|
||||
except Exception as e: # noqa: BLE001
|
||||
raise SystemExit(f"[!] 读取 bundles 失败: {e}")
|
||||
bundles = [b for b in file_bundles if isinstance(b, dict)]
|
||||
for b in bundles:
|
||||
for pb in validate_bundle(b):
|
||||
print(f"[套件] 警告 {b.get('ownerId', '?')}: {pb}", file=sys.stderr)
|
||||
|
||||
if args.url:
|
||||
captured = capture_bundles(
|
||||
args.url, user_data_dir=args.user_data_dir, channel=args.channel,
|
||||
cdp_url=args.connect, owner_display=args.owner_display or "",
|
||||
wait_seconds=args.wait_seconds)
|
||||
bundles = merge_bundles(bundles, captured)
|
||||
if args.save_bundles:
|
||||
with open(args.save_bundles, "w", encoding="utf-8") as f:
|
||||
json.dump(bundles, f, ensure_ascii=False, indent=2)
|
||||
print(f"[套件] 已保存 {len(bundles)} 个所有者套件 -> {args.save_bundles}")
|
||||
|
||||
if not bundles:
|
||||
raise SystemExit("[!] 没有可用套件：请提供 --url 在线捕获，或 --bundles 离线回放。")
|
||||
|
||||
if not args.names:
|
||||
print("[i] 未提供 --names：仅完成套件捕获/校验，不执行查询。")
|
||||
return
|
||||
|
||||
try:
|
||||
names = load_names(args.names)
|
||||
except Exception as e: # noqa: BLE001
|
||||
raise SystemExit(f"[!] 名单读取失败: {e}")
|
||||
if not names:
|
||||
raise SystemExit(f"[!] 名单为空或无法解析：{args.names}"
|
||||
"（支持 json/txt/csv/xlsx，列名用 群组名称/group_name）")
|
||||
|
||||
print(f"[回放] {len(names)} 个名字 × {len(bundles)} 个所有者，{args.max_workers} 线程……")
|
||||
rows = []
|
||||
with ThreadPoolExecutor(max_workers=args.max_workers) as ex:
|
||||
futs = [ex.submit(resolve, n, bundles) for n in names]
|
||||
for done, f in enumerate(as_completed(futs), 1):
|
||||
rows.append(f.result())
|
||||
if done % 50 == 0 or done == len(names):
|
||||
print(f"[回放] {done}/{len(names)}")
|
||||
|
||||
order = {n: i for i, n in enumerate(names)}
|
||||
rows.sort(key=lambda r: order.get(r[0], 10 ** 9))
|
||||
out_path = write_out(args.out, RESULT_HEADER, rows)
|
||||
|
||||
hit = sum(1 for r in rows if r[3])
|
||||
print(f"[输出] 精确命中 {hit}/{len(rows)} -> {out_path}")
|
||||
misses = [r[0] for r in rows if not r[3]]
|
||||
if misses:
|
||||
preview = "、".join(misses[:10]) + ("……" if len(misses) > 10 else "")
|
||||
print(f"[复核] 未命中 {len(misses)} 个（候选见备注列）: {preview}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user