dan/dchadwick
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updates for scraper errors, still erroring sometimes though.

Browse Source
This commit is contained in:
Dan Chadwick 2024-09-21 10:53:38 -04:00
parent 57da5fed63
commit 41d0d557fb
3 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scripts/update_everything.sh
View File

@ -1,8 +1,7 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# import all fighters. # import all fighters.
drush @self.local ev '\Drupal::service("ufc.importer")->importFighters()';
ddev drush @self.local ev '\Drupal::service("ufc.import_fighters")->importFighters();'; ddev drush @self.local ev '\Drupal::service("ufc.import_fighters")->importFighters();';
ddev drush @self.local ev '\Drupal::service("ufc.import_fights")->importEvents();'; ddev drush @self.local ev '\Drupal::service("ufc.import_fights")->importEvents();';
ddev drush @self.local ev '\Drupal::service("ufc.import_fights")->createEvents();'; ddev drush @self.local ev '\Drupal::service("ufc.import_fights")->createFights();';

6
web/modules/custom/ufc/src/Fighter.php
View File

@ -186,7 +186,13 @@ public function setFighterRecord(): void {
*/ */
public function setAccuracy(): void { public function setAccuracy(): void {
$circles = $this->crawler->filter('.e-chart-circle'); $circles = $this->crawler->filter('.e-chart-circle');
if (empty($circles)) {
return;
}
foreach ($circles as $circle) { foreach ($circles as $circle) {
if (is_null($circle)) {
continue;
}
foreach ($circle->childNodes as $child) { foreach ($circle->childNodes as $child) {
$text = strtolower($child->textContent); $text = strtolower($child->textContent);
if (str_contains($text, "accuracy")) { if (str_contains($text, "accuracy")) {

4
web/modules/custom/ufc/src/Services/FighterImporter.php
View File

@ -174,8 +174,8 @@ public function loopThroughFighterPages($base_url): void {
/* $cid = "ufc:" . $url; */ /* $cid = "ufc:" . $url; */
$request = $this->httpClient->request('GET', $url, $headers); $request = $this->httpClient->request('GET', $url, $headers);
$content = $request->getBody()->getContents(); $content = $request->getBody()->getContents();
$check = strpos($content, "No Result Found For"); $invalid_page = strpos($content, "No Result Found For");
if (!$check) { if (!$invalid_page) {
\Drupal::logger('ufc')->notice("Extracting fighters from page $i."); \Drupal::logger('ufc')->notice("Extracting fighters from page $i.");
self::extractFighters($content); self::extractFighters($content);
continue; continue;