Coverage for src/pyrf_data/client.py: 49%
293 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-09-06 21:54 +0000
1"""Client for accessing Ringfinity match data sources.
3Wraps the FootyIQ watcher data API and the bet3 ratings API.
4"""
6from __future__ import annotations
8import json
9import os
10import re
11from dataclasses import dataclass
12from pathlib import Path
13from typing import Any, cast
15import pandas as pd
16import requests
17from dotenv import load_dotenv
19# Default date range constants
20DEFAULT_FROM_DATE = pd.Timestamp("2000-07-07")
21DEFAULT_TO_DATE = pd.Timestamp("2029-08-07")
23# Filenames within the local data directory
24WCPLAYER_EXPORT_FILENAME = "wcplayerexport.json"
25WC2026_PLAYERS_FILENAME = "wc2026_players_flashscore.parquet"
27# FlashScore 2026 World Cup squads article
28WC2026_SQUADS_URL = "https://www.flashscore.com/news/soccer-world-cup-official-full-squads-fifa-world-cup/UsBRzJS9/"
30_FS_POSITION_LABEL_TO_CODE = {
31 "Goalkeepers": "GK",
32 "Defenders": "DF",
33 "Midfielders": "MF",
34 "Forwards": "FW",
35 "Attackers": "FW",
36 "Strikers": "FW",
37 "Reserves": "RES",
38}
39_FS_POSITION_LABELS_RE = re.compile(
40 r"^\s*(" + "|".join(_FS_POSITION_LABEL_TO_CODE) + r")\b\s*:?\s*",
41 re.IGNORECASE,
42)
43_FS_TEAM_HREF_RE = re.compile(r"/team/(?P<slug>[^/]+)/(?P<fs_id>[^/]+)/?")
44_FS_PLAYER_HREF_RE = re.compile(r"/player/(?P<slug>[^/]+)/(?P<fs_id>[^/]+)/?")
45_FS_INVISIBLE_RE = re.compile(r"[\u200b\u200c\u200d\u202c\u202d\u00ad]")
46_FS_GROUP_HEADING_RE = re.compile(r"^Group\s+[A-L]$", re.IGNORECASE)
49def _timestamp_to_str(ts: pd.Timestamp) -> str:
50 """Convert a pandas Timestamp to YYYY-MM-DD string format."""
51 return ts.strftime("%Y-%m-%d")
54def _flatten_scores_response(data: list[dict[str, Any]]) -> pd.DataFrame:
55 """Flatten nested scores response into a DataFrame with one row per fixture.
57 The API returns data nested as: country -> comps -> seasons -> rounds -> fixtures.
58 This function flattens it to a tabular format.
59 """
60 rows: list[dict[str, Any]] = []
62 for country_data in data:
63 country = country_data.get("country", "")
65 for comp in country_data.get("comps", []):
66 comp_id = comp.get("id")
67 comp_name = comp.get("name", "")
68 comp_short = comp.get("shortName", "")
70 for season in comp.get("seasons", []):
71 season_name = season.get("name", "")
73 for round_data in season.get("rounds", []):
74 round_name = round_data.get("name", "")
76 for fixture in round_data.get("fixtures", []):
77 row = {
78 "fixture_id": fixture.get("id"),
79 "start_time": pd.to_datetime(fixture.get("startTime")),
80 "country": country,
81 "comp_id": comp_id,
82 "comp_name": comp_name,
83 "comp_short": comp_short,
84 "season": season_name,
85 "round": round_name,
86 "team1_id": (fixture.get("team1") or {}).get("_id"),
87 "team1_name": (fixture.get("team1") or {}).get("name", ""),
88 "team2_id": (fixture.get("team2") or {}).get("_id"),
89 "team2_name": (fixture.get("team2") or {}).get("name", ""),
90 "score_ft": fixture.get("scoreFT", ""),
91 "score_ht": fixture.get("scoreHT", ""),
92 "team1_goals": (fixture.get("bestScore") or {}).get("team1"),
93 "team2_goals": (fixture.get("bestScore") or {}).get("team2"),
94 "scores_agree": (fixture.get("bestScore") or {}).get("scoresAgree"),
95 "pitch": ((fixture.get("watcher") or {}).get("conditions") or {}).get("pitch"),
96 "weather": ((fixture.get("watcher") or {}).get("conditions") or {}).get("weather"),
97 "temp": ((fixture.get("watcher") or {}).get("conditions") or {}).get("temp"),
98 }
99 rows.append(row)
101 return pd.DataFrame(rows)
104def _flatten_full_export(data: dict[str, Any]) -> pd.DataFrame:
105 """Flatten full export response into a DataFrame with one row per fixture.
107 Includes detailed match statistics for each team.
108 """
109 rows: list[dict[str, Any]] = []
111 for fixture_data in data.get("fixtures", []):
112 fixture = fixture_data.get("fixture") or {}
113 match_data = fixture_data.get("matchData") or {}
114 conditions = fixture_data.get("conditions") or {}
115 league = fixture.get("league") or {}
117 team1_match = match_data.get("team1") or {}
118 team2_match = match_data.get("team2") or {}
119 team1_stats = team1_match.get("stats") or {}
120 team2_stats = team2_match.get("stats") or {}
122 start_time_str = fixture.get("startTime")
123 row = {
124 # Fixture info
125 "fixture_id": fixture.get("_id"),
126 "start_time": pd.to_datetime(start_time_str) if start_time_str else pd.NaT,
127 "status": fixture_data.get("status"),
128 # League info
129 "country": league.get("country", ""),
130 "comp_id": league.get("_id"),
131 "comp_name": league.get("name", ""),
132 "comp_short": league.get("shortName", ""),
133 "season": league.get("season", ""),
134 "round_num": fixture.get("roundNum"),
135 "is_cup": fixture.get("isCup", False),
136 # Team info
137 "team1_id": (fixture.get("team1") or {}).get("_id"),
138 "team1_name": (fixture.get("team1") or {}).get("name", ""),
139 "team2_id": (fixture.get("team2") or {}).get("_id"),
140 "team2_name": (fixture.get("team2") or {}).get("name", ""),
141 # Scores
142 "score_ft": fixture.get("scoreFT", ""),
143 "score_ht": fixture.get("scoreHT", ""),
144 # Conditions
145 "pitch": conditions.get("pitch"),
146 "weather": conditions.get("weather"),
147 "temp": conditions.get("temp"),
148 # Team 1 stats
149 "team1_goals": (team1_stats.get("goals") or {}).get("total"),
150 "team1_goals_h1": (team1_stats.get("goals") or {}).get("firstHalf"),
151 "team1_goals_h2": (team1_stats.get("goals") or {}).get("secondHalf"),
152 "team1_chances": (team1_stats.get("chances") or {}).get("total"),
153 "team1_chances_h1": (team1_stats.get("chances") or {}).get("firstHalf"),
154 "team1_chances_h2": (team1_stats.get("chances") or {}).get("secondHalf"),
155 "team1_half_chances": (team1_stats.get("halfChances") or {}).get("total"),
156 "team1_half_chances_h1": (team1_stats.get("halfChances") or {}).get("firstHalf"),
157 "team1_half_chances_h2": (team1_stats.get("halfChances") or {}).get("secondHalf"),
158 "team1_deliveries": (team1_stats.get("deliveries") or {}).get("total"),
159 "team1_deliveries_h1": (team1_stats.get("deliveries") or {}).get("firstHalf"),
160 "team1_deliveries_h2": (team1_stats.get("deliveries") or {}).get("secondHalf"),
161 "team1_ooohs": (team1_stats.get("ooohs") or {}).get("total"),
162 "team1_choos": (team1_stats.get("choos") or {}).get("total"),
163 # Team 2 stats
164 "team2_goals": (team2_stats.get("goals") or {}).get("total"),
165 "team2_goals_h1": (team2_stats.get("goals") or {}).get("firstHalf"),
166 "team2_goals_h2": (team2_stats.get("goals") or {}).get("secondHalf"),
167 "team2_chances": (team2_stats.get("chances") or {}).get("total"),
168 "team2_chances_h1": (team2_stats.get("chances") or {}).get("firstHalf"),
169 "team2_chances_h2": (team2_stats.get("chances") or {}).get("secondHalf"),
170 "team2_half_chances": (team2_stats.get("halfChances") or {}).get("total"),
171 "team2_half_chances_h1": (team2_stats.get("halfChances") or {}).get("firstHalf"),
172 "team2_half_chances_h2": (team2_stats.get("halfChances") or {}).get("secondHalf"),
173 "team2_deliveries": (team2_stats.get("deliveries") or {}).get("total"),
174 "team2_deliveries_h1": (team2_stats.get("deliveries") or {}).get("firstHalf"),
175 "team2_deliveries_h2": (team2_stats.get("deliveries") or {}).get("secondHalf"),
176 "team2_ooohs": (team2_stats.get("ooohs") or {}).get("total"),
177 "team2_choos": (team2_stats.get("choos") or {}).get("total"),
178 }
179 rows.append(row)
181 return pd.DataFrame(rows)
184def _flatten_wcplayer_export(raw: dict[str, Any]) -> pd.DataFrame:
185 """Flatten a wcplayerexport payload into a one-row-per-player DataFrame."""
186 df = pd.json_normalize(raw.get("wcteamplayers", []))
187 for col in ("fs", "ng", "tm"):
188 if col in df.columns and any(c.startswith(f"{col}.") for c in df.columns):
189 df = df.drop(columns=[col])
190 for col in ("createdAt", "updatedAt", "v1StarRatingChangedAt", "v2StarRatingChangedAt"):
191 if col in df.columns:
192 df[col] = pd.to_datetime(df[col])
193 return df
196@dataclass
197class PyRfDataClient:
198 """Client for the FootyIQ watcher data API.
200 Returns DataFrames with flattened data by default. Use `*_raw` methods
201 for the original nested JSON response.
202 """
204 watcher_token: str | None = None
205 ratings_token: str | None = None
206 watcher_url: str = "https://portal.footyiq.co.uk/api/watcher"
207 ratings_url: str = "https://g.bet3.co.uk/api"
208 rf_local_path: Path | str | None = None
210 def __post_init__(self) -> None:
211 """Initialize tokens and paths from environment if not provided."""
212 if self.watcher_token is None or self.ratings_token is None or self.rf_local_path is None:
213 load_dotenv()
214 if self.watcher_token is None:
215 self.watcher_token = os.environ.get("FOOTYIQ_TOKEN")
216 if self.ratings_token is None:
217 self.ratings_token = os.environ.get("BET3_TOKEN")
218 if self.rf_local_path is None:
219 env_path = os.environ.get("RF_LOCAL_PATH")
220 self.rf_local_path = Path(env_path) if env_path else None
221 if self.rf_local_path is not None:
222 self.rf_local_path = Path(self.rf_local_path)
224 @property
225 def _watcher_headers(self) -> dict[str, str]:
226 """Build authorization headers for watcher API requests."""
227 return {"Authorization": f"Bearer {self.watcher_token}"}
229 @property
230 def _ratings_headers(self) -> dict[str, str]:
231 """Build authorization headers for ratings API requests."""
232 return {"Authorization": f"Bearer {self.ratings_token}"}
234 # -------------------------------------------------------------------------
235 # Raw JSON methods
236 # -------------------------------------------------------------------------
238 def fetch_scores_raw(
239 self,
240 limit: int = 1000,
241 from_date: pd.Timestamp | None = None,
242 to_date: pd.Timestamp | None = None,
243 scores_agree: bool | None = None,
244 ) -> list[dict[str, Any]]:
245 """Fetch game scores from the API as raw nested JSON.
247 Args:
248 limit: Maximum number of records to fetch.
249 from_date: Start date as pd.Timestamp (defaults to 2000-07-07).
250 to_date: End date as pd.Timestamp (defaults to 2029-08-07).
251 scores_agree: If True, only return games where watcher and NowGoal agree.
252 If False, only return games where they disagree.
253 If None, return all games regardless.
255 Returns:
256 Raw nested JSON response (country -> comps -> seasons -> rounds -> fixtures).
257 """
258 from_date = from_date or DEFAULT_FROM_DATE
259 to_date = to_date or DEFAULT_TO_DATE
261 url = f"{self.watcher_url}/export/scoresonly"
262 params: dict[str, Any] = {
263 "limit": limit,
264 "from": _timestamp_to_str(from_date),
265 "to": _timestamp_to_str(to_date),
266 }
267 if scores_agree is not None:
268 params["scoresAgree"] = str(scores_agree).lower()
270 response = requests.get(url, params=params, headers=self._watcher_headers, timeout=60)
271 response.raise_for_status()
272 return cast(list[dict[str, Any]], response.json())
274 def fetch_full_export_raw(
275 self,
276 limit: int = 500000,
277 from_date: pd.Timestamp | None = None,
278 to_date: pd.Timestamp | None = None,
279 ) -> dict[str, Any]:
280 """Fetch full export data as raw JSON.
282 Returns:
283 Raw JSON with 'fixtures' list and metadata.
284 """
285 from_date = from_date or DEFAULT_FROM_DATE
286 to_date = to_date or DEFAULT_TO_DATE
288 url = f"{self.watcher_url}/export"
289 params: dict[str, Any] = {
290 "limit": limit,
291 "from": _timestamp_to_str(from_date),
292 "to": _timestamp_to_str(to_date),
293 }
294 response = requests.get(url, params=params, headers=self._watcher_headers, timeout=120)
295 response.raise_for_status()
296 return cast(dict[str, Any], response.json())
298 # -------------------------------------------------------------------------
299 # DataFrame methods
300 # -------------------------------------------------------------------------
302 def fetch_scores(
303 self,
304 limit: int = 1000,
305 from_date: pd.Timestamp | None = None,
306 to_date: pd.Timestamp | None = None,
307 scores_agree: bool | None = None,
308 ) -> pd.DataFrame:
309 """Fetch game scores as a flattened DataFrame.
311 Args:
312 limit: Maximum number of records to fetch.
313 from_date: Start date as pd.Timestamp (defaults to 2000-07-07).
314 to_date: End date as pd.Timestamp (defaults to 2029-08-07).
315 scores_agree: If True, only return games where watcher and NowGoal agree.
316 If False, only return games where they disagree.
317 If None, return all games regardless.
319 Returns:
320 DataFrame with columns: fixture_id, start_time, country, comp_id, comp_name,
321 comp_short, season, round, team1_id, team1_name, team2_id, team2_name,
322 score_ft, score_ht, team1_goals, team2_goals, scores_agree, pitch,
323 weather, temp.
324 """
325 raw_data = self.fetch_scores_raw(
326 limit=limit,
327 from_date=from_date,
328 to_date=to_date,
329 scores_agree=scores_agree,
330 )
331 return _flatten_scores_response(raw_data)
333 def fetch_scores_agreed(
334 self,
335 limit: int = 500000,
336 from_date: pd.Timestamp | None = None,
337 to_date: pd.Timestamp | None = None,
338 ) -> pd.DataFrame:
339 """Fetch all trusted data where watcher and NowGoal scores agree.
341 This is the recommended method for fetching reliable historical data.
343 Returns:
344 DataFrame with one row per fixture.
345 """
346 return self.fetch_scores(
347 limit=limit,
348 from_date=from_date,
349 to_date=to_date,
350 scores_agree=True,
351 )
353 def fetch_scores_disagreed(
354 self,
355 limit: int = 1000,
356 from_date: pd.Timestamp | None = None,
357 to_date: pd.Timestamp | None = None,
358 ) -> pd.DataFrame:
359 """Fetch games where watcher score differs from NowGoal score.
361 Useful for investigating data quality or discrepancies.
363 Returns:
364 DataFrame with one row per fixture.
365 """
366 return self.fetch_scores(
367 limit=limit,
368 from_date=from_date,
369 to_date=to_date,
370 scores_agree=False,
371 )
373 def fetch_full_export(
374 self,
375 limit: int = 500000,
376 from_date: pd.Timestamp | None = None,
377 to_date: pd.Timestamp | None = None,
378 ) -> pd.DataFrame:
379 """Fetch full export with detailed match statistics as DataFrame.
381 Includes per-team stats: goals, chances, half-chances, deliveries,
382 ooohs, choos - broken down by half.
384 Returns:
385 DataFrame with detailed match statistics.
386 """
387 raw_data = self.fetch_full_export_raw(
388 limit=limit,
389 from_date=from_date,
390 to_date=to_date,
391 )
392 return _flatten_full_export(raw_data)
394 # -------------------------------------------------------------------------
395 # Ratings
396 # -------------------------------------------------------------------------
398 def fetch_ratings_club_raw(self) -> list[dict[str, Any]]:
399 """Fetch club team ratings as raw JSON.
401 Returns:
402 List of rating objects, one per home/away club matchup, each
403 containing alpha/beta/lambda parameters, supremacy (sup), total
404 goals (tg), and team/league metadata.
405 """
406 url = f"{self.ratings_url}/numbers/ratings"
407 response = requests.get(url, headers=self._ratings_headers, timeout=60)
408 response.raise_for_status()
409 return cast(list[dict[str, Any]], response.json())
411 def fetch_ratings_club(self) -> pd.DataFrame:
412 """Fetch club team ratings as a DataFrame.
414 Returns:
415 DataFrame with one row per club matchup, including alpha/beta/lambda
416 parameters, supremacy, total goals, and team/league metadata.
417 The ``updated_at`` column is parsed to datetime.
418 """
419 raw_data = self.fetch_ratings_club_raw()
420 df = pd.DataFrame(raw_data)
421 if "updated_at" in df.columns:
422 df["updated_at"] = pd.to_datetime(df["updated_at"])
423 return df
425 def fetch_ratings_international_raw(self) -> list[dict[str, Any]]:
426 """Fetch international (national-team) ratings as raw JSON.
428 Returns:
429 List of rating objects, one per nation, each containing supremacy
430 (sup), total goals (tg, goals), per-side goal expectations,
431 normalised alpha/beta, and team/NowGoal metadata.
432 """
433 url = f"{self.ratings_url}/numbers/ratings_mark"
434 response = requests.get(url, headers=self._ratings_headers, timeout=60)
435 response.raise_for_status()
436 return cast(list[dict[str, Any]], response.json())
438 def fetch_ratings_international(self) -> pd.DataFrame:
439 """Fetch international (national-team) ratings as a DataFrame.
441 Returns:
442 DataFrame with one row per nation, including supremacy, total goals,
443 per-side goal expectations, normalised alpha/beta, and team/NowGoal
444 metadata. The ``updated_at`` column is parsed to datetime.
445 """
446 raw_data = self.fetch_ratings_international_raw()
447 df = pd.DataFrame(raw_data)
448 if "updated_at" in df.columns:
449 df["updated_at"] = pd.to_datetime(df["updated_at"])
450 return df
452 # -------------------------------------------------------------------------
453 # Local wcplayer export
454 # -------------------------------------------------------------------------
456 def load_wcplayer_export_raw(self) -> dict[str, Any]:
457 """Load the local ``wcplayerexport.json`` file as raw JSON.
459 Returns:
460 Dict with keys ``wcteamplayers`` (list of player records),
461 ``exportedAt``, and ``count``.
462 """
463 if self.rf_local_path is None:
464 msg = (
465 "rf_local_path is not set. Provide it via the constructor "
466 "or set the RF_LOCAL_PATH environment variable."
467 )
468 raise ValueError(msg)
469 path = Path(self.rf_local_path) / WCPLAYER_EXPORT_FILENAME
470 with path.open("r", encoding="utf-8") as f:
471 return cast(dict[str, Any], json.load(f))
473 def load_wcplayer_export(self) -> pd.DataFrame:
474 """Load the local ``wcplayerexport.json`` as a flat DataFrame.
476 Nested ``fs`` / ``ng`` / ``tm`` sub-objects are flattened with
477 dot-separated column names (e.g. ``tm.cpm``, ``ng._id``).
478 Date columns (``createdAt``, ``updatedAt``, ``v1StarRatingChangedAt``,
479 ``v2StarRatingChangedAt``) are parsed to datetime.
481 Returns:
482 DataFrame with one row per player.
483 """
484 return _flatten_wcplayer_export(self.load_wcplayer_export_raw())
486 def fetch_wcplayer_export_raw(self, limit: int = 5000) -> dict[str, Any]:
487 """Fetch the wcteamplayers export from the API as raw JSON.
489 Returns:
490 Dict with keys ``wcteamplayers`` (list of player records),
491 ``exportedAt``, and ``count``.
492 """
493 url = f"{self.watcher_url}/export/wcteamplayers"
494 params: dict[str, Any] = {"limit": limit}
495 response = requests.get(url, params=params, headers=self._watcher_headers, timeout=600)
496 response.raise_for_status()
497 return cast(dict[str, Any], response.json())
499 def fetch_wcplayer_export(self, limit: int = 5000, *, cache: bool = True) -> pd.DataFrame:
500 """Fetch the wcteamplayers export from the API as a flat DataFrame.
502 When ``cache`` is True (default) and ``rf_local_path`` is set, the raw
503 JSON response is written to ``<rf_local_path>/wcplayerexport.json``,
504 overwriting any existing file.
506 Returns:
507 DataFrame with one row per player, matching ``load_wcplayer_export``.
508 """
509 raw = self.fetch_wcplayer_export_raw(limit=limit)
510 if cache and self.rf_local_path is not None:
511 path = Path(self.rf_local_path) / WCPLAYER_EXPORT_FILENAME
512 path.parent.mkdir(parents=True, exist_ok=True)
513 with path.open("w", encoding="utf-8") as f:
514 json.dump(raw, f)
515 return _flatten_wcplayer_export(raw)
517 # -------------------------------------------------------------------------
518 # FlashScore 2026 World Cup squads
519 # -------------------------------------------------------------------------
521 def _wc2026_cache_path(self, cache_path: Path | str | None) -> Path:
522 """Resolve the parquet cache location for the WC2026 squads."""
523 if cache_path is not None:
524 return Path(cache_path)
525 if self.rf_local_path is not None:
526 return Path(self.rf_local_path) / WC2026_PLAYERS_FILENAME
527 return Path("data") / WC2026_PLAYERS_FILENAME
529 def fetch_wc2026_squads(
530 self,
531 *,
532 update: bool = False,
533 cache_path: Path | str | None = None,
534 url: str = WC2026_SQUADS_URL,
535 ) -> pd.DataFrame:
536 """Return the 2026 World Cup squads as one row per (nation, player).
538 Reads ``data/wc2026_players_flashscore.parquet`` (or ``rf_local_path`` equivalent)
539 by default. Pass ``update=True`` to re-scrape the FlashScore article
540 at ``url`` and overwrite the cache.
542 See ``docs/specs/wc2026_v2.md`` for the column contract.
543 """
544 path = self._wc2026_cache_path(cache_path)
545 if not update:
546 if not path.exists():
547 msg = (
548 f"WC2026 squad cache not found at {path}. "
549 "Call fetch_wc2026_squads(update=True) once to populate it."
550 )
551 raise FileNotFoundError(msg)
552 return pd.read_parquet(path)
554 response = requests.get(
555 url,
556 headers={"User-Agent": "pyrf-notebook/2.0"},
557 timeout=60,
558 )
559 response.raise_for_status()
560 df = _parse_wc2026_squads_html(response.text)
561 path.parent.mkdir(parents=True, exist_ok=True)
562 df.to_parquet(path, index=False)
563 return df
566def _clean_fs_text(text: str) -> str:
567 """Strip zero-width / bidi control chars and surrounding whitespace."""
568 return _FS_INVISIBLE_RE.sub("", text).strip()
571def _extract_first_paren(tail: str) -> str | None:
572 """Return the first parenthesised club name from a FlashScore tail."""
573 match = re.search(r"\(([^)]+)\)", tail)
574 if not match:
575 return None
576 inner = match.group(1).strip()
577 # Drop "on loan from ...", trailing notes, etc.
578 inner = inner.split(",")[0].strip()
579 return inner or None
582def _parse_wc2026_squads_html(html: str) -> pd.DataFrame:
583 """Parse the FlashScore squads article into a per-player DataFrame.
585 See ``docs/specs/wc2026_v2.md`` § 3 for the algorithm and column contract.
586 """
587 from bs4 import BeautifulSoup, NavigableString, Tag
589 soup = BeautifulSoup(html, "html.parser")
590 scraped_at = pd.Timestamp.now("UTC").tz_localize(None)
592 # Walk every heading + paragraph in document order. Anchoring to a
593 # specific article container is fragile; the heading text itself
594 # (`Group A` ... `Related Articles`) gives us a clean window.
595 elements = soup.find_all(["h1", "h2", "h3", "h4", "p"])
597 rows: list[dict[str, Any]] = []
598 nations_seen: dict[str, dict[str, Any]] = {}
600 current_group: str | None = None
601 current_nation: str | None = None
602 current_fs_country: str | None = None
603 current_fs_team_id: str | None = None
604 current_position_label: str | None = None
605 nation_has_players: dict[str, bool] = {}
606 nation_status_hint: dict[str, str] = {}
608 in_window = False
610 def _emit_status_for(nation: str) -> str:
611 """Classify a nation's squad status from observed players and hints."""
612 if nation_has_players.get(nation):
613 return "final"
614 return nation_status_hint.get(nation, "pending")
616 for el in elements:
617 text = _clean_fs_text(el.get_text(" ", strip=True))
618 if not text:
619 continue
621 # Window control: start at "Group A", stop at "Related Articles".
622 if not in_window:
623 if _FS_GROUP_HEADING_RE.match(text):
624 in_window = True
625 else:
626 continue
627 elif text.lower().startswith("related articles"):
628 break
630 tag_name = el.name
632 if tag_name in {"h1", "h2", "h3", "h4"}:
633 if _FS_GROUP_HEADING_RE.match(text):
634 current_group = text
635 continue
636 # Nation headings carry a /team/<slug>/<id>/ anchor.
637 team_anchor: Tag | None = None
638 team_match: re.Match[str] | None = None
639 for a in el.find_all("a"):
640 if not isinstance(a, Tag):
641 continue
642 href = a.get("href") or ""
643 m = _FS_TEAM_HREF_RE.search(str(href))
644 if m is not None:
645 team_anchor = a
646 team_match = m
647 break
648 if team_anchor is None or team_match is None:
649 # Non-nation heading inside the window (e.g. "MENTIONS").
650 current_position_label = None
651 continue
652 current_nation = _clean_fs_text(team_anchor.get_text(" ", strip=True))
653 current_fs_country = team_match.group("slug")
654 current_fs_team_id = team_match.group("fs_id")
655 current_position_label = None
656 nations_seen.setdefault(
657 current_nation,
658 {
659 "nation": current_nation,
660 "fs_country": current_fs_country,
661 "fs_team_id": current_fs_team_id,
662 "group": current_group,
663 },
664 )
665 nation_has_players.setdefault(current_nation, False)
666 continue
668 if tag_name != "p" or current_nation is None:
669 continue
671 # Strip and record a leading "Goalkeepers:" / "Defenders:" / ... label.
672 label_match = _FS_POSITION_LABELS_RE.match(text)
673 body_html_parts: list[Any] = list(el.children)
674 if label_match:
675 current_position_label = label_match.group(1).title()
677 # Look for status hints when the paragraph has no player anchors.
678 anchors = [
679 a for a in el.find_all("a") if isinstance(a, Tag) and _FS_PLAYER_HREF_RE.search(str(a.get("href") or ""))
680 ]
681 if not anchors:
682 low = text.lower()
683 if "preliminary" in low and current_nation not in nation_status_hint:
684 nation_status_hint[current_nation] = "preliminary"
685 elif "will announce" in low and current_nation not in nation_status_hint:
686 nation_status_hint[current_nation] = "pending"
687 continue
689 # Walk children to keep player → following-text association intact.
690 # We collect (anchor, trailing_text_until_next_anchor) pairs.
691 pairs: list[tuple[Tag, str]] = []
692 pending: Tag | None = None
693 trailing_buf: list[str] = []
694 for child in body_html_parts:
695 if isinstance(child, Tag) and child.name == "a" and _FS_PLAYER_HREF_RE.search(str(child.get("href") or "")):
696 if pending is not None:
697 pairs.append((pending, "".join(trailing_buf)))
698 pending = child
699 trailing_buf = []
700 else:
701 if isinstance(child, NavigableString):
702 trailing_buf.append(str(child))
703 elif isinstance(child, Tag):
704 trailing_buf.append(child.get_text(" "))
705 if pending is not None:
706 pairs.append((pending, "".join(trailing_buf)))
708 for anchor, tail in pairs:
709 match = _FS_PLAYER_HREF_RE.search(str(anchor.get("href") or ""))
710 if match is None:
711 continue
712 fs_id = match.group("fs_id")
713 fs_slug = match.group("slug")
714 fs_name = _clean_fs_text(anchor.get_text(" ", strip=True))
715 fs_club = _extract_first_paren(tail)
716 rows.append(
717 {
718 "nation": current_nation,
719 "fs_country": current_fs_country,
720 "fs_team_id": current_fs_team_id,
721 "fs_position": _FS_POSITION_LABEL_TO_CODE.get(current_position_label or ""),
722 "fs_position_label": current_position_label,
723 "fs_name": fs_name,
724 "fs_slug": fs_slug,
725 "fs_id": fs_id,
726 "fs_club": fs_club,
727 "fs_squad_status": "final",
728 "scraped_at": scraped_at,
729 }
730 )
731 nation_has_players[current_nation] = True
733 # Placeholder rows for nations declared but without any players parsed.
734 for nation, meta in nations_seen.items():
735 if nation_has_players.get(nation):
736 continue
737 status = _emit_status_for(nation)
738 rows.append(
739 {
740 "nation": nation,
741 "fs_country": meta["fs_country"],
742 "fs_team_id": meta["fs_team_id"],
743 "fs_position": None,
744 "fs_position_label": None,
745 "fs_name": None,
746 "fs_slug": None,
747 "fs_id": None,
748 "fs_club": None,
749 "fs_squad_status": status,
750 "scraped_at": scraped_at,
751 }
752 )
754 df = pd.DataFrame(rows)
755 if not df.empty:
756 # Stamp the final status onto every row of each nation.
757 status_per_nation = {nation: _emit_status_for(nation) for nation in nations_seen}
758 df["fs_squad_status"] = df["nation"].map(status_per_nation).fillna(df["fs_squad_status"])
759 return df