transinthesouth #1
13 changed files with 438 additions and 15 deletions
2
.env
2
.env
|
@ -0,0 +1,2 @@
|
||||||
|
APP_ENV=dev
|
||||||
|
MESSENGER_TRANSPORT_DSN=doctrine://default
|
|
@ -11,9 +11,6 @@
|
||||||
# DO NOT DEFINE PRODUCTION SECRETS IN THIS FILE NOR IN ANY OTHER COMMITTED FILES.
|
# DO NOT DEFINE PRODUCTION SECRETS IN THIS FILE NOR IN ANY OTHER COMMITTED FILES.
|
||||||
# https://symfony.com/doc/current/configuration/secrets.html
|
# https://symfony.com/doc/current/configuration/secrets.html
|
||||||
|
|
||||||
APP_ENV=dev
|
|
||||||
MESSENGER_TRANSPORT_DSN=doctrine://default
|
|
||||||
|
|
||||||
# Generate your own value with php -r "echo bin2hex(random_bytes(16)) . PHP_EOL;"
|
# Generate your own value with php -r "echo bin2hex(random_bytes(16)) . PHP_EOL;"
|
||||||
APP_SECRET=TODO
|
APP_SECRET=TODO
|
||||||
|
|
||||||
|
|
|
@ -22,7 +22,9 @@
|
||||||
"simpod/doctrine-utcdatetime": "^0.2.0",
|
"simpod/doctrine-utcdatetime": "^0.2.0",
|
||||||
"symfony/apache-pack": "^1.0",
|
"symfony/apache-pack": "^1.0",
|
||||||
"symfony/console": "6.2.*",
|
"symfony/console": "6.2.*",
|
||||||
|
"symfony/css-selector": "6.2.*",
|
||||||
"symfony/doctrine-messenger": "6.2.*",
|
"symfony/doctrine-messenger": "6.2.*",
|
||||||
|
"symfony/dom-crawler": "6.2.*",
|
||||||
"symfony/dotenv": "6.2.*",
|
"symfony/dotenv": "6.2.*",
|
||||||
"symfony/flex": "^2",
|
"symfony/flex": "^2",
|
||||||
"symfony/framework-bundle": "6.2.*",
|
"symfony/framework-bundle": "6.2.*",
|
||||||
|
|
206
composer.lock
generated
206
composer.lock
generated
|
@ -4,7 +4,7 @@
|
||||||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||||
"This file is @generated automatically"
|
"This file is @generated automatically"
|
||||||
],
|
],
|
||||||
"content-hash": "7a39e53db29b1bb2087ed0a9f6c52e51",
|
"content-hash": "ec60a70816b9a72f22376bc30fd698a2",
|
||||||
"packages": [
|
"packages": [
|
||||||
{
|
{
|
||||||
"name": "doctrine/cache",
|
"name": "doctrine/cache",
|
||||||
|
@ -1620,6 +1620,75 @@
|
||||||
],
|
],
|
||||||
"time": "2022-12-08T02:08:23+00:00"
|
"time": "2022-12-08T02:08:23+00:00"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "masterminds/html5",
|
||||||
|
"version": "2.7.6",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/Masterminds/html5-php.git",
|
||||||
|
"reference": "897eb517a343a2281f11bc5556d6548db7d93947"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/Masterminds/html5-php/zipball/897eb517a343a2281f11bc5556d6548db7d93947",
|
||||||
|
"reference": "897eb517a343a2281f11bc5556d6548db7d93947",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"ext-ctype": "*",
|
||||||
|
"ext-dom": "*",
|
||||||
|
"ext-libxml": "*",
|
||||||
|
"php": ">=5.3.0"
|
||||||
|
},
|
||||||
|
"require-dev": {
|
||||||
|
"phpunit/phpunit": "^4.8.35 || ^5.7.21 || ^6 || ^7"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"extra": {
|
||||||
|
"branch-alias": {
|
||||||
|
"dev-master": "2.7-dev"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"Masterminds\\": "src"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Matt Butcher",
|
||||||
|
"email": "technosophos@gmail.com"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Matt Farina",
|
||||||
|
"email": "matt@mattfarina.com"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Asmir Mustafic",
|
||||||
|
"email": "goetas@gmail.com"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "An HTML5 parser and serializer.",
|
||||||
|
"homepage": "http://masterminds.github.io/html5-php",
|
||||||
|
"keywords": [
|
||||||
|
"HTML5",
|
||||||
|
"dom",
|
||||||
|
"html",
|
||||||
|
"parser",
|
||||||
|
"querypath",
|
||||||
|
"serializer",
|
||||||
|
"xml"
|
||||||
|
],
|
||||||
|
"support": {
|
||||||
|
"issues": "https://github.com/Masterminds/html5-php/issues",
|
||||||
|
"source": "https://github.com/Masterminds/html5-php/tree/2.7.6"
|
||||||
|
},
|
||||||
|
"time": "2022-08-18T16:18:26+00:00"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "meteo-concept/hcaptcha-bundle",
|
"name": "meteo-concept/hcaptcha-bundle",
|
||||||
"version": "v3.3.0",
|
"version": "v3.3.0",
|
||||||
|
@ -2811,6 +2880,71 @@
|
||||||
],
|
],
|
||||||
"time": "2022-12-28T14:26:22+00:00"
|
"time": "2022-12-28T14:26:22+00:00"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "symfony/css-selector",
|
||||||
|
"version": "v6.2.5",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/symfony/css-selector.git",
|
||||||
|
"reference": "bf1b9d4ad8b1cf0dbde8b08e0135a2f6259b9ba1"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/symfony/css-selector/zipball/bf1b9d4ad8b1cf0dbde8b08e0135a2f6259b9ba1",
|
||||||
|
"reference": "bf1b9d4ad8b1cf0dbde8b08e0135a2f6259b9ba1",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"php": ">=8.1"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"Symfony\\Component\\CssSelector\\": ""
|
||||||
|
},
|
||||||
|
"exclude-from-classmap": [
|
||||||
|
"/Tests/"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Fabien Potencier",
|
||||||
|
"email": "fabien@symfony.com"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Jean-François Simon",
|
||||||
|
"email": "jeanfrancois.simon@sensiolabs.com"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Symfony Community",
|
||||||
|
"homepage": "https://symfony.com/contributors"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "Converts CSS selectors to XPath expressions",
|
||||||
|
"homepage": "https://symfony.com",
|
||||||
|
"support": {
|
||||||
|
"source": "https://github.com/symfony/css-selector/tree/v6.2.5"
|
||||||
|
},
|
||||||
|
"funding": [
|
||||||
|
{
|
||||||
|
"url": "https://symfony.com/sponsor",
|
||||||
|
"type": "custom"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"url": "https://github.com/fabpot",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
|
||||||
|
"type": "tidelift"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"time": "2023-01-01T08:38:09+00:00"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "symfony/dependency-injection",
|
"name": "symfony/dependency-injection",
|
||||||
"version": "v6.2.3",
|
"version": "v6.2.3",
|
||||||
|
@ -3152,6 +3286,76 @@
|
||||||
],
|
],
|
||||||
"time": "2022-11-04T07:42:34+00:00"
|
"time": "2022-11-04T07:42:34+00:00"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "symfony/dom-crawler",
|
||||||
|
"version": "v6.2.5",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/symfony/dom-crawler.git",
|
||||||
|
"reference": "19aa4962a0687e96941f0bdb27b794c5b73e2394"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/symfony/dom-crawler/zipball/19aa4962a0687e96941f0bdb27b794c5b73e2394",
|
||||||
|
"reference": "19aa4962a0687e96941f0bdb27b794c5b73e2394",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"masterminds/html5": "^2.6",
|
||||||
|
"php": ">=8.1",
|
||||||
|
"symfony/polyfill-ctype": "~1.8",
|
||||||
|
"symfony/polyfill-mbstring": "~1.0"
|
||||||
|
},
|
||||||
|
"require-dev": {
|
||||||
|
"symfony/css-selector": "^5.4|^6.0"
|
||||||
|
},
|
||||||
|
"suggest": {
|
||||||
|
"symfony/css-selector": ""
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"Symfony\\Component\\DomCrawler\\": ""
|
||||||
|
},
|
||||||
|
"exclude-from-classmap": [
|
||||||
|
"/Tests/"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Fabien Potencier",
|
||||||
|
"email": "fabien@symfony.com"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Symfony Community",
|
||||||
|
"homepage": "https://symfony.com/contributors"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "Eases DOM navigation for HTML and XML documents",
|
||||||
|
"homepage": "https://symfony.com",
|
||||||
|
"support": {
|
||||||
|
"source": "https://github.com/symfony/dom-crawler/tree/v6.2.5"
|
||||||
|
},
|
||||||
|
"funding": [
|
||||||
|
{
|
||||||
|
"url": "https://symfony.com/sponsor",
|
||||||
|
"type": "custom"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"url": "https://github.com/fabpot",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
|
||||||
|
"type": "tidelift"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"time": "2023-01-20T17:45:48+00:00"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "symfony/dotenv",
|
"name": "symfony/dotenv",
|
||||||
"version": "v6.2.0",
|
"version": "v6.2.0",
|
||||||
|
|
|
@ -5,8 +5,10 @@ namespace App\Command;
|
||||||
use App\DataSource\DataSourceException;
|
use App\DataSource\DataSourceException;
|
||||||
use App\DataSource\DataSourceInterface;
|
use App\DataSource\DataSourceInterface;
|
||||||
use App\DataSource\ErinReedDataSource;
|
use App\DataSource\ErinReedDataSource;
|
||||||
|
use App\DataSource\TransInTheSouthDataSource;
|
||||||
use App\Entity\Clinic;
|
use App\Entity\Clinic;
|
||||||
use App\Entity\ImportHash;
|
use App\Entity\ImportHash;
|
||||||
|
use App\HereMaps\Client as HereClient;
|
||||||
use App\Repository\ClinicRepository;
|
use App\Repository\ClinicRepository;
|
||||||
use App\Repository\ImportHashRepository;
|
use App\Repository\ImportHashRepository;
|
||||||
use Symfony\Component\Console\Attribute\AsCommand;
|
use Symfony\Component\Console\Attribute\AsCommand;
|
||||||
|
@ -23,19 +25,23 @@ use Symfony\Contracts\HttpClient\HttpClientInterface;
|
||||||
class ClinicsImportCommand extends Command
|
class ClinicsImportCommand extends Command
|
||||||
{
|
{
|
||||||
private HttpClientInterface $httpClient;
|
private HttpClientInterface $httpClient;
|
||||||
|
private HereClient $hereClient;
|
||||||
private ClinicRepository $clinics;
|
private ClinicRepository $clinics;
|
||||||
private ImportHashRepository $imports;
|
private ImportHashRepository $imports;
|
||||||
|
|
||||||
private array $dataSources = [
|
private array $dataSources = [
|
||||||
ErinReedDataSource::class
|
ErinReedDataSource::class,
|
||||||
|
TransInTheSouthDataSource::class,
|
||||||
];
|
];
|
||||||
|
|
||||||
public function __construct(
|
public function __construct(
|
||||||
HttpClientInterface $httpClient,
|
HttpClientInterface $httpClient,
|
||||||
|
HereClient $hereClient,
|
||||||
ClinicRepository $clinics,
|
ClinicRepository $clinics,
|
||||||
ImportHashRepository $imports,
|
ImportHashRepository $imports,
|
||||||
) {
|
) {
|
||||||
$this->httpClient = $httpClient;
|
$this->httpClient = $httpClient;
|
||||||
|
$this->hereClient = $hereClient;
|
||||||
$this->clinics = $clinics;
|
$this->clinics = $clinics;
|
||||||
$this->imports = $imports;
|
$this->imports = $imports;
|
||||||
parent::__construct();
|
parent::__construct();
|
||||||
|
@ -49,7 +55,7 @@ class ClinicsImportCommand extends Command
|
||||||
$clinicsAddedCount = 0;
|
$clinicsAddedCount = 0;
|
||||||
foreach ($this->dataSources as $source) {
|
foreach ($this->dataSources as $source) {
|
||||||
/* @var DataSourceInterface $source */
|
/* @var DataSourceInterface $source */
|
||||||
$source = new $source($this->httpClient);
|
$source = new $source($this->httpClient, $this->hereClient);
|
||||||
|
|
||||||
$io->section($source->getType());
|
$io->section($source->getType());
|
||||||
$io->text('Fetching clinics');
|
$io->text('Fetching clinics');
|
||||||
|
@ -72,6 +78,7 @@ class ClinicsImportCommand extends Command
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$source->preImport($new);
|
||||||
$this->clinics->save($new);
|
$this->clinics->save($new);
|
||||||
|
|
||||||
$import = new ImportHash();
|
$import = new ImportHash();
|
||||||
|
|
|
@ -14,7 +14,7 @@ use Symfony\Component\Console\Output\OutputInterface;
|
||||||
use Symfony\Component\Console\Style\SymfonyStyle;
|
use Symfony\Component\Console\Style\SymfonyStyle;
|
||||||
|
|
||||||
#[AsCommand(
|
#[AsCommand(
|
||||||
name: 'app:reset',
|
name: 'app:clinics:reset',
|
||||||
description: 'Dev tool used to reset the application database',
|
description: 'Dev tool used to reset the application database',
|
||||||
)]
|
)]
|
||||||
class ResetCommand extends Command
|
class ResetCommand extends Command
|
||||||
|
|
|
@ -21,6 +21,7 @@ use EasyCorp\Bundle\EasyAdminBundle\Filter\ChoiceFilter;
|
||||||
use EasyCorp\Bundle\EasyAdminBundle\Router\AdminUrlGenerator;
|
use EasyCorp\Bundle\EasyAdminBundle\Router\AdminUrlGenerator;
|
||||||
use Oefenweb\DamerauLevenshtein\DamerauLevenshtein as Levenshtein;
|
use Oefenweb\DamerauLevenshtein\DamerauLevenshtein as Levenshtein;
|
||||||
use Symfony\Component\HttpFoundation\RedirectResponse;
|
use Symfony\Component\HttpFoundation\RedirectResponse;
|
||||||
|
use Symfony\Component\Validator\Constraints\Choice;
|
||||||
|
|
||||||
class ClinicCrudController extends AbstractCrudController
|
class ClinicCrudController extends AbstractCrudController
|
||||||
{
|
{
|
||||||
|
@ -76,7 +77,12 @@ class ClinicCrudController extends AbstractCrudController
|
||||||
->add(ChoiceFilter::new('published')->setChoices([
|
->add(ChoiceFilter::new('published')->setChoices([
|
||||||
'No' => false,
|
'No' => false,
|
||||||
'Yes' => true
|
'Yes' => true
|
||||||
]));
|
]))
|
||||||
|
->add(ChoiceFilter::new('dataSource')->canSelectMultiple()->setChoices([
|
||||||
|
'Trans in the South' => 'transInTheSouth',
|
||||||
|
'Erin Reed' => 'erinReed',
|
||||||
|
]))
|
||||||
|
;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function configureFields(string $pageName): iterable
|
public function configureFields(string $pageName): iterable
|
||||||
|
|
|
@ -3,16 +3,19 @@
|
||||||
namespace App\DataSource;
|
namespace App\DataSource;
|
||||||
|
|
||||||
use App\Entity\Clinic;
|
use App\Entity\Clinic;
|
||||||
|
use App\HereMaps\Client;
|
||||||
use Symfony\Contracts\HttpClient\HttpClientInterface;
|
use Symfony\Contracts\HttpClient\HttpClientInterface;
|
||||||
|
|
||||||
interface DataSourceInterface
|
interface DataSourceInterface
|
||||||
{
|
{
|
||||||
public const DATASOURCE__ERIN_REED = 'erinReed';
|
public const DATASOURCE__ERIN_REED = 'erinReed';
|
||||||
|
public const DATASOURCE__TRANS_IN_THE_SOUTH = 'transInTheSouth';
|
||||||
public const DATASOURCE__MANUAL_ENTRY = 'manualEntry';
|
public const DATASOURCE__MANUAL_ENTRY = 'manualEntry';
|
||||||
|
|
||||||
public function __construct(HttpClientInterface $httpClient);
|
public function __construct(HttpClientInterface $httpClient, Client $hereClient);
|
||||||
public function getType(): string;
|
public function getType(): string;
|
||||||
/* @throws DataSourceException */
|
/* @throws DataSourceException */
|
||||||
public function fetchClinics(): array;
|
public function fetchClinics(): array;
|
||||||
public function hash(Clinic $clinic): string;
|
public function hash(Clinic $clinic): string;
|
||||||
|
public function preImport(Clinic $clinic): void;
|
||||||
}
|
}
|
|
@ -3,6 +3,7 @@
|
||||||
namespace App\DataSource;
|
namespace App\DataSource;
|
||||||
|
|
||||||
use App\Entity\Clinic;
|
use App\Entity\Clinic;
|
||||||
|
use App\HereMaps\Client as HereClient;
|
||||||
use Symfony\Contracts\HttpClient\HttpClientInterface;
|
use Symfony\Contracts\HttpClient\HttpClientInterface;
|
||||||
|
|
||||||
class ErinReedDataSource implements DataSourceInterface
|
class ErinReedDataSource implements DataSourceInterface
|
||||||
|
@ -11,8 +12,10 @@ class ErinReedDataSource implements DataSourceInterface
|
||||||
|
|
||||||
private HttpClientInterface $httpClient;
|
private HttpClientInterface $httpClient;
|
||||||
|
|
||||||
public function __construct(HttpClientInterface $httpClient)
|
public function __construct(
|
||||||
{
|
HttpClientInterface $httpClient,
|
||||||
|
HereClient $hereClient,
|
||||||
|
) {
|
||||||
$this->httpClient = $httpClient;
|
$this->httpClient = $httpClient;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -103,4 +106,9 @@ class ErinReedDataSource implements DataSourceInterface
|
||||||
$data = implode( '.', $pieces );
|
$data = implode( '.', $pieces );
|
||||||
return md5( $data );
|
return md5( $data );
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function preImport(Clinic $clinic): void
|
||||||
|
{
|
||||||
|
// left intentionally blank
|
||||||
|
}
|
||||||
}
|
}
|
159
src/DataSource/TransInTheSouthDataSource.php
Normal file
159
src/DataSource/TransInTheSouthDataSource.php
Normal file
|
@ -0,0 +1,159 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\DataSource;
|
||||||
|
|
||||||
|
use App\Entity\Clinic;
|
||||||
|
use App\HereMaps\Client as HereClient;
|
||||||
|
use Symfony\Component\DomCrawler\Crawler;
|
||||||
|
use Symfony\Contracts\HttpClient\HttpClientInterface;
|
||||||
|
|
||||||
|
class TransInTheSouthDataSource implements DataSourceInterface {
|
||||||
|
|
||||||
|
private HttpClientInterface $httpClient;
|
||||||
|
private HereClient $hereClient;
|
||||||
|
|
||||||
|
public function __construct(
|
||||||
|
HttpClientInterface $httpClient,
|
||||||
|
HereClient $hereClient,
|
||||||
|
) {
|
||||||
|
$this->httpClient = $httpClient;
|
||||||
|
$this->hereClient = $hereClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getType(): string
|
||||||
|
{
|
||||||
|
return self::DATASOURCE__TRANS_IN_THE_SOUTH;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @throws DataSourceException
|
||||||
|
*/
|
||||||
|
public function fetchClinics(): array
|
||||||
|
{
|
||||||
|
$tisProvidersId = $this->_scrapeTisProvidersId();
|
||||||
|
$html = $this->_loadSearchResults($tisProvidersId);
|
||||||
|
$rawRecords = $this->_scrapeData($html);
|
||||||
|
|
||||||
|
$clinics = [];
|
||||||
|
foreach ($rawRecords as $record) {
|
||||||
|
$clinic = new Clinic();
|
||||||
|
$clinic->setName($record['name']);
|
||||||
|
$clinic->setDescription($record['description']);
|
||||||
|
$clinic->setAddress($record['address']);
|
||||||
|
$clinic->setDataSource($this->getType());
|
||||||
|
$clinic->setPublished(false);
|
||||||
|
$clinics[] = $clinic;
|
||||||
|
}
|
||||||
|
|
||||||
|
return $clinics;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function _scrapeTisProvidersId(): string
|
||||||
|
{
|
||||||
|
try {
|
||||||
|
$res = $this->httpClient->request('GET', 'https://southernequality.org/resources/transinthesouth/');
|
||||||
|
$html = $res->getContent();
|
||||||
|
} catch (\Throwable) {
|
||||||
|
throw new DataSourceException('HTTP request to fetch search form failed.', $this->getType());
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($html === '') {
|
||||||
|
throw new DataSourceException('Missing web page content in response.', $this->getType());
|
||||||
|
}
|
||||||
|
|
||||||
|
$crawler = new Crawler($html);
|
||||||
|
return $crawler->filter('#filter-tis-providers')->first()->attr('value');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @throws DataSourceException
|
||||||
|
*/
|
||||||
|
private function _loadSearchResults(string $tisProvidersId): string
|
||||||
|
{
|
||||||
|
try {
|
||||||
|
$res = $this->httpClient->request('POST', 'https://southernequality.org/resources/transinthesouth/', [
|
||||||
|
'body' => 'tis-name-search=&states=&services%5B%5D=Informed+Consent&services%5B%5D=Offers+Hormone+Replacement+Therapy+%28HRT%29&filter-tis-providers=' . $tisProvidersId . '&_wp_http_referer=%2Fresources%2Ftransinthesouth%2F&filter_providers=Search',
|
||||||
|
]);
|
||||||
|
$html = $res->getContent();
|
||||||
|
} catch (\Throwable) {
|
||||||
|
throw new DataSourceException('HTTP request to fetch search results failed.', $this->getType());
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($html === '') {
|
||||||
|
throw new DataSourceException('Missing web page content in response.', $this->getType());
|
||||||
|
}
|
||||||
|
|
||||||
|
return $html;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @throws DataSourceException
|
||||||
|
*/
|
||||||
|
private function _scrapeData(string $html): array
|
||||||
|
{
|
||||||
|
$rawRecords = [];
|
||||||
|
|
||||||
|
$crawler = new Crawler($html);
|
||||||
|
$crawler->filter('.provider')->each(function(Crawler $crawler) use (&$rawRecords) {
|
||||||
|
$services = [];
|
||||||
|
foreach ($crawler->filter('.accordion-header') as $service) {
|
||||||
|
$services[] = $service->textContent;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!in_array('Offers Hormone Replacement Therapy (HRT)', $services) || !in_array('Informed Consent', $services)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
$name = $crawler->filter('.provider--title')->text();
|
||||||
|
if (!$name) {
|
||||||
|
throw new DataSourceException('Missing clinic attribute in scraped data: name', $this->getType());
|
||||||
|
}
|
||||||
|
|
||||||
|
$practice = $crawler->filter('.provider--practice-name')->text();
|
||||||
|
if ($practice && ($name !== $practice)) {
|
||||||
|
$name .= ' - ' . $practice;
|
||||||
|
}
|
||||||
|
|
||||||
|
$description = $crawler->filter('.provider--summary')->text();
|
||||||
|
|
||||||
|
$address = $crawler->filter('.provider--address')->text();
|
||||||
|
if (!$address) {
|
||||||
|
throw new DataSourceException('Missing clinic attribute in scraped data: address', $this->getType());
|
||||||
|
}
|
||||||
|
|
||||||
|
$rawRecords[] = [
|
||||||
|
'name' => $name,
|
||||||
|
'description' => $description,
|
||||||
|
'address' => $address,
|
||||||
|
];
|
||||||
|
});
|
||||||
|
|
||||||
|
return $rawRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function hash(Clinic $clinic): string
|
||||||
|
{
|
||||||
|
$pieces = [
|
||||||
|
$clinic->getName(),
|
||||||
|
$clinic->getDescription(),
|
||||||
|
$clinic->getAddress(),
|
||||||
|
];
|
||||||
|
$data = implode('.', $pieces);
|
||||||
|
return md5($data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @throws \Exception
|
||||||
|
* @throws DataSourceException
|
||||||
|
*/
|
||||||
|
public function preImport(Clinic $clinic): void
|
||||||
|
{
|
||||||
|
$items = $this->hereClient->geocode($clinic->getAddress())['items'];
|
||||||
|
if (count($items) === 0) {
|
||||||
|
throw new DataSourceException('No coordinates found for address: ' . $clinic->getAddress(), $this->getType());
|
||||||
|
}
|
||||||
|
|
||||||
|
$clinic->setLatitude($items[0]['position']['lat']);
|
||||||
|
$clinic->setLongitude($items[0]['position']['lng']);
|
||||||
|
}
|
||||||
|
}
|
|
@ -46,4 +46,30 @@ class Client {
|
||||||
|
|
||||||
return $decoded;
|
return $decoded;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @throws \Exception
|
||||||
|
*/
|
||||||
|
public function geocode(string $address): array
|
||||||
|
{
|
||||||
|
$url = 'https://geocode.search.hereapi.com/v1/geocode?' . http_build_query([
|
||||||
|
'lang' => 'en-US',
|
||||||
|
'q' => $address,
|
||||||
|
'apiKey' => $this->hereApiKey,
|
||||||
|
]);
|
||||||
|
|
||||||
|
try {
|
||||||
|
$res = $this->httpClient->request('GET', $url);
|
||||||
|
$data = $res->getContent();
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
throw new \Exception('HTTP request to Here Maps failed: ' . $e->getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
$decoded = json_decode($data, true);
|
||||||
|
if ($decoded === false) {
|
||||||
|
throw new \Exception('Failed to decode Here Maps response');
|
||||||
|
}
|
||||||
|
|
||||||
|
return $decoded;
|
||||||
|
}
|
||||||
}
|
}
|
|
@ -24,6 +24,7 @@ Parameters:
|
||||||
<div class="d-grid">
|
<div class="d-grid">
|
||||||
{{ form_widget(searchForm.submit, {'attr': {'class': 'rounded-pill btn-primary'}}) }}
|
{{ form_widget(searchForm.submit, {'attr': {'class': 'rounded-pill btn-primary'}}) }}
|
||||||
</div>
|
</div>
|
||||||
|
{{ form_widget(searchForm.page, {'attr': {'value': '1'}}) }}
|
||||||
</div>
|
</div>
|
||||||
{{ form_end(searchForm) }}
|
{{ form_end(searchForm) }}
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -49,12 +49,20 @@
|
||||||
{% if clinic.address %}
|
{% if clinic.address %}
|
||||||
<p class="mb-2">{{ clinic.address }}</p>
|
<p class="mb-2">{{ clinic.address }}</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% if clinic.dataSource == "erinReed" %}
|
{% if clinic.dataSource != 'manualEntry' %}
|
||||||
<span class="badge bg-blue text-dark-blue" style="color: rgb(0, 77, 124);">
|
<span class="badge text-bg-secondary me-2">
|
||||||
<i class="fa-solid fa-check"></i>
|
<i class="fa-solid fa-database"></i>
|
||||||
Informed Consent
|
{% if clinic.dataSource == "erinReed" %}
|
||||||
|
<a class="text-white" target="_blank" href="https://www.google.com/maps/d/viewer?mid=1DxyOTw8dI8n96BHFF2JVUMK7bXsRKtzA&ll=41.639103490264155%2C-83.26303679041781&z=3">Erin Reed</a>
|
||||||
|
{% elseif clinic.dataSource == 'transInTheSouth' %}
|
||||||
|
<a class="text-white" target="_blank" href="https://southernequality.org/resources/transinthesouth/">Trans in the South</a>
|
||||||
|
{% endif %}
|
||||||
</span>
|
</span>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
<span class="badge bg-blue text-dark-blue">
|
||||||
|
<i class="fa-solid fa-check"></i>
|
||||||
|
Informed Consent
|
||||||
|
</span>
|
||||||
{% if clinic.description %}
|
{% if clinic.description %}
|
||||||
<hr>
|
<hr>
|
||||||
<p class="mb-0">{{ clinic.description|raw }}</p>
|
<p class="mb-0">{{ clinic.description|raw }}</p>
|
||||||
|
|
Loading…
Reference in a new issue