dan/dchadwick
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More updates to import from ufc.com json api.

Browse Source
This commit is contained in:
dan612 2025-12-07 12:54:42 -05:00
parent 8cee39a6df
commit 034dc92721
4 changed files with 108 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
web/modules/custom/ufc/src/Commands/UfcCommands.php
View File

@ -5,6 +5,7 @@
use Drupal\ufc\Services\FighterImporter; use Drupal\ufc\Services\FighterImporter;
use Drupal\ufc\Services\FightImporter; use Drupal\ufc\Services\FightImporter;
use Drupal\Core\Batch\BatchBuilder; use Drupal\Core\Batch\BatchBuilder;
use Drupal\Core\Cache\CacheBackendInterface;
use Drupal\node\Entity\Node; use Drupal\node\Entity\Node;
use Drush\Commands\DrushCommands; use Drush\Commands\DrushCommands;
use Drush\Attributes as CLI; use Drush\Attributes as CLI;
@ -12,10 +13,13 @@
class UfcCommands extends DrushCommands { class UfcCommands extends DrushCommands {
/**
* The UFC fighter list cache ID.
*/
protected $cacheId = 'ufc:fighter-list'; protected $cacheId = 'ufc:fighter-list';
/** /**
* Import fighters from UFC.com. * Import fighters from UFC.com JSON:API.
*/ */
#[CLI\Command(name: 'ufc:import-fighters', aliases: ['impft'])] #[CLI\Command(name: 'ufc:import-fighters', aliases: ['impft'])]
public function importFighters(): void { public function importFighters(): void {
@ -24,30 +28,20 @@ public function importFighters(): void {
// First check for the item in cache. // First check for the item in cache.
$fighter_list = \Drupal::cache()->get($this->cacheId); $fighter_list = \Drupal::cache()->get($this->cacheId);
if (!$fighter_list) { if (!$fighter_list) {
/* $fighter_list = $fighter_importer->getListOfCurrentFighters(); */ dump("Getting fighters from UFC.com JSON:API");
$api_list = \Drupal::httpClient()->get('https://www.ufc.com/jsonapi/node/athlete')->getBody()->getContents(); $fighter_list = $fighter_importer->getListOfCurrentFighters();
$fighter_list = json_decode($api_list)->data; \Drupal::cache()->set($this->cacheId, $fighter_list, CacheBackendInterface::CACHE_PERMANENT);
dump($fighter_list);
exit();
\Drupal::cache()->set($this->cacheId, $fighter_list, time() + 300);
} }
else { else {
$fighter_list = $fighter_list->data; $fighter_list = $fighter_list->data;
} }
$fighter_import_queue = \Drupal::queue('fighter_import'); $fighter_import_queue = \Drupal::queue('fighter_import');
foreach ($fighter_list as $division => $fighters) { foreach ($fighter_list as $fighter) {
foreach ($fighters as $fighter) { $queue_item = new \stdClass();
$queue_item = new \stdClass(); $queue_item->id = $fighter['id'];
$queue_item->first_name = $fighter['firstname']; $fighter_import_queue->createItem($queue_item);
$queue_item->last_name = $fighter['lastname'];
$queue_item->image = $fighter['image'];
$queue_item->division = $division;
$queue_item->profile = $fighter['profile'];
$fighter_import_queue->createItem($queue_item);
}
} }
} }
/** /**

38
web/modules/custom/ufc/src/Plugin/QueueWorker/FighterQueueWorker.php
View File

@ -95,28 +95,22 @@ public function processItem($data): void {
* The item to process. * The item to process.
*/ */
private function doProcess($item): bool { private function doProcess($item): bool {
$fighter = new Fighter(\Drupal::httpClient()); // @todo:
$fighter->first_name = $item->first_name; // Item->id will be the ID of the fighter.
$fighter->last_name = $item->last_name; // Need to fetch fighter list from cache.
$fighter->image = $item->image; // Then get the corred fighter based on the id.
$fighter->class = $item->division; // Then update or create the fighter node, including any media.
if (!$fighter->scrapeDataFromFighterPage($item->profile)) { // these values will need to be retrieves from the existing data.
\Drupal::logger('ufc')->alert("FAILED: $fighter->first_name $fighter->last_name to " . $item->profile); /* $node_lookup = reset(\Drupal::entityTypeManager()->getStorage('node')->loadByProperties(['title' => $title])); */
} /* if (!empty($node_lookup)) { */
// Check if node exists, by title. /* // Update instead of create. */
$fighter->createMediaEntityFromImage(); /* $fighter->updatePlayerNode($node_lookup->id()); */
$title = $fighter->first_name . " " . $fighter->last_name; /* \Drupal::logger('ufc')->notice("$title updated successfully."); */
$node_lookup = reset(\Drupal::entityTypeManager()->getStorage('node')->loadByProperties(['title' => $title])); /* } */
/* else { */
if (!empty($node_lookup)) { /* \Drupal::logger('ufc')->warning("No existing player found for $title...creating"); */
// Update instead of create. /* $fighter->createPlayerNode(); */
$fighter->updatePlayerNode($node_lookup->id()); /* } */
\Drupal::logger('ufc')->notice("$title updated successfully.");
}
else {
\Drupal::logger('ufc')->warning("No existing player found for $title...creating");
$fighter->createPlayerNode();
}
return TRUE; return TRUE;
} }

88
web/modules/custom/ufc/src/Services/FighterImporter.php
View File

@ -24,22 +24,18 @@ class FighterImporter {
*/ */
protected $entityTypeManager; protected $entityTypeManager;
/**
* The UFC cache bin.
*/
protected $cache;
/** /**
* Array of all fighters. * Array of all fighters.
* @var array[] * @var array[]
*/ */
public $fighters = []; public $fighters = [];
/** public $cookieJar;
* The current weight class.
* @var string
*/
protected $weightClass;
/**
* The UFC cache bin.
*/
protected $cache;
/** /**
* The base url for fighter lists. * The base url for fighter lists.
@ -64,6 +60,64 @@ public function __construct(
$this->cache = $cache; $this->cache = $cache;
} }
/**
* Get list of current fighters.
*
* @return array $fighters
*/
public function getListOfCurrentFighters(): array {
$this->addFightersFromUrl(self::UFC_BASE);
return $this->fighters;
}
/**
* Add fighters from a URL.
*
* @param string $url
* The URL to scrape.
*/
public function addFightersFromUrl(string $url) {
sleep(10);
dump("Requesting: " . $url);
$proxied_url = 'http://192.168.0.13:8888/?url=' . urlencode($url);
$ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Safari/605.1.15';
$headers = [
'referer' => true,
'verify' => false,
'headers' => [
'User-Agent' => $ua,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding' => 'gzip, deflate, br',
'Cache-Control' => 'no-cache',
'Pragma' => 'no-cache',
'Referer' => 'https://www.ufc.com/athletes/all',
]
];
$response = $this->httpClient->request('GET', $proxied_url, $headers);
$content = $response->getBody()->getContents();
$content_arr = json_decode($content, TRUE);
if (!$content_arr) {
exit("something went wrong");
}
$athletes = $content_arr['data'];
$count = count($athletes);
dump("Found " . $count . " fighters.");
foreach ($athletes as $athlete) {
$this->fighters[] = $athlete;
}
$next_page = $content_arr['links']['next']['href'] ?? FALSE;
if ($next_page) {
$this->addFightersFromUrl($next_page);
}
}
/**
* There is a pager, loop through to get all fighters.
*
* @param string $base_url
*/
public function importFighters(): void { public function importFighters(): void {
// Get JSON File from API. // Get JSON File from API.
$api_athletes = $this->cache->get('ufc_api_athletes'); $api_athletes = $this->cache->get('ufc_api_athletes');
@ -75,20 +129,6 @@ public function importFighters(): void {
// Iterate over data to import all fighters. // Iterate over data to import all fighters.
} }
/**
* Get list of current fighters.
*
* @return array $fighters
*/
public function getListOfCurrentFighters(): array {
foreach ($this->divisions as $division => $div_base_url) {
$division_url = self::UFC_BASE . $div_base_url;
$this->weightClass = $division;
echo "Starting import for " . $division . "\n";
self::loopThroughFighterPages($division_url);
}
return $this->fighters;
}
/** /**
* There is a pager, loop through to get all fighters. * There is a pager, loop through to get all fighters.

32
web/sites/development.services.yml
View File

@ -12,19 +12,19 @@
# This means that if you want to override any value of a parameter, the # This means that if you want to override any value of a parameter, the
# whole parameter array needs to be copied from # whole parameter array needs to be copied from
# sites/default/default.services.yml or from core/core.services.yml file. # sites/default/default.services.yml or from core/core.services.yml file.
# parameters: parameters:
# http.response.debug_cacheability_headers: true http.response.debug_cacheability_headers: true
# services: services:
# cache.backend.null: cache.backend.null:
# class: Drupal\Core\Cache\NullBackendFactory class: Drupal\Core\Cache\NullBackendFactory
# logger.channel.config_schema: logger.channel.config_schema:
# parent: logger.channel_base parent: logger.channel_base
# arguments: [ 'config_schema' ] arguments: [ 'config_schema' ]
# config.schema_checker: config.schema_checker:
# class: Drupal\Core\Config\Development\LenientConfigSchemaChecker class: Drupal\Core\Config\Development\LenientConfigSchemaChecker
# arguments: arguments:
# - '@config.typed' - '@config.typed'
# - '@messenger' - '@messenger'
# - '@logger.channel.config_schema' - '@logger.channel.config_schema'
# tags: tags:
# - { name: event_subscriber } - { name: event_subscriber }