transinthesouth #1

Merged
alena merged 8 commits from transinthesouth into main 2023-02-17 02:32:53 +00:00
13 changed files with 438 additions and 15 deletions

2
.env
View file

@ -0,0 +1,2 @@
APP_ENV=dev
MESSENGER_TRANSPORT_DSN=doctrine://default

View file

@ -11,9 +11,6 @@
# DO NOT DEFINE PRODUCTION SECRETS IN THIS FILE NOR IN ANY OTHER COMMITTED FILES. # DO NOT DEFINE PRODUCTION SECRETS IN THIS FILE NOR IN ANY OTHER COMMITTED FILES.
# https://symfony.com/doc/current/configuration/secrets.html # https://symfony.com/doc/current/configuration/secrets.html
APP_ENV=dev
MESSENGER_TRANSPORT_DSN=doctrine://default
# Generate your own value with php -r "echo bin2hex(random_bytes(16)) . PHP_EOL;" # Generate your own value with php -r "echo bin2hex(random_bytes(16)) . PHP_EOL;"
APP_SECRET=TODO APP_SECRET=TODO

View file

@ -22,7 +22,9 @@
"simpod/doctrine-utcdatetime": "^0.2.0", "simpod/doctrine-utcdatetime": "^0.2.0",
"symfony/apache-pack": "^1.0", "symfony/apache-pack": "^1.0",
"symfony/console": "6.2.*", "symfony/console": "6.2.*",
"symfony/css-selector": "6.2.*",
"symfony/doctrine-messenger": "6.2.*", "symfony/doctrine-messenger": "6.2.*",
"symfony/dom-crawler": "6.2.*",
"symfony/dotenv": "6.2.*", "symfony/dotenv": "6.2.*",
"symfony/flex": "^2", "symfony/flex": "^2",
"symfony/framework-bundle": "6.2.*", "symfony/framework-bundle": "6.2.*",

206
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically" "This file is @generated automatically"
], ],
"content-hash": "7a39e53db29b1bb2087ed0a9f6c52e51", "content-hash": "ec60a70816b9a72f22376bc30fd698a2",
"packages": [ "packages": [
{ {
"name": "doctrine/cache", "name": "doctrine/cache",
@ -1620,6 +1620,75 @@
], ],
"time": "2022-12-08T02:08:23+00:00" "time": "2022-12-08T02:08:23+00:00"
}, },
{
"name": "masterminds/html5",
"version": "2.7.6",
"source": {
"type": "git",
"url": "https://github.com/Masterminds/html5-php.git",
"reference": "897eb517a343a2281f11bc5556d6548db7d93947"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/Masterminds/html5-php/zipball/897eb517a343a2281f11bc5556d6548db7d93947",
"reference": "897eb517a343a2281f11bc5556d6548db7d93947",
"shasum": ""
},
"require": {
"ext-ctype": "*",
"ext-dom": "*",
"ext-libxml": "*",
"php": ">=5.3.0"
},
"require-dev": {
"phpunit/phpunit": "^4.8.35 || ^5.7.21 || ^6 || ^7"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "2.7-dev"
}
},
"autoload": {
"psr-4": {
"Masterminds\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Matt Butcher",
"email": "technosophos@gmail.com"
},
{
"name": "Matt Farina",
"email": "matt@mattfarina.com"
},
{
"name": "Asmir Mustafic",
"email": "goetas@gmail.com"
}
],
"description": "An HTML5 parser and serializer.",
"homepage": "http://masterminds.github.io/html5-php",
"keywords": [
"HTML5",
"dom",
"html",
"parser",
"querypath",
"serializer",
"xml"
],
"support": {
"issues": "https://github.com/Masterminds/html5-php/issues",
"source": "https://github.com/Masterminds/html5-php/tree/2.7.6"
},
"time": "2022-08-18T16:18:26+00:00"
},
{ {
"name": "meteo-concept/hcaptcha-bundle", "name": "meteo-concept/hcaptcha-bundle",
"version": "v3.3.0", "version": "v3.3.0",
@ -2811,6 +2880,71 @@
], ],
"time": "2022-12-28T14:26:22+00:00" "time": "2022-12-28T14:26:22+00:00"
}, },
{
"name": "symfony/css-selector",
"version": "v6.2.5",
"source": {
"type": "git",
"url": "https://github.com/symfony/css-selector.git",
"reference": "bf1b9d4ad8b1cf0dbde8b08e0135a2f6259b9ba1"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/css-selector/zipball/bf1b9d4ad8b1cf0dbde8b08e0135a2f6259b9ba1",
"reference": "bf1b9d4ad8b1cf0dbde8b08e0135a2f6259b9ba1",
"shasum": ""
},
"require": {
"php": ">=8.1"
},
"type": "library",
"autoload": {
"psr-4": {
"Symfony\\Component\\CssSelector\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Fabien Potencier",
"email": "fabien@symfony.com"
},
{
"name": "Jean-François Simon",
"email": "jeanfrancois.simon@sensiolabs.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Converts CSS selectors to XPath expressions",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/css-selector/tree/v6.2.5"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2023-01-01T08:38:09+00:00"
},
{ {
"name": "symfony/dependency-injection", "name": "symfony/dependency-injection",
"version": "v6.2.3", "version": "v6.2.3",
@ -3152,6 +3286,76 @@
], ],
"time": "2022-11-04T07:42:34+00:00" "time": "2022-11-04T07:42:34+00:00"
}, },
{
"name": "symfony/dom-crawler",
"version": "v6.2.5",
"source": {
"type": "git",
"url": "https://github.com/symfony/dom-crawler.git",
"reference": "19aa4962a0687e96941f0bdb27b794c5b73e2394"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/dom-crawler/zipball/19aa4962a0687e96941f0bdb27b794c5b73e2394",
"reference": "19aa4962a0687e96941f0bdb27b794c5b73e2394",
"shasum": ""
},
"require": {
"masterminds/html5": "^2.6",
"php": ">=8.1",
"symfony/polyfill-ctype": "~1.8",
"symfony/polyfill-mbstring": "~1.0"
},
"require-dev": {
"symfony/css-selector": "^5.4|^6.0"
},
"suggest": {
"symfony/css-selector": ""
},
"type": "library",
"autoload": {
"psr-4": {
"Symfony\\Component\\DomCrawler\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Fabien Potencier",
"email": "fabien@symfony.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Eases DOM navigation for HTML and XML documents",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/dom-crawler/tree/v6.2.5"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2023-01-20T17:45:48+00:00"
},
{ {
"name": "symfony/dotenv", "name": "symfony/dotenv",
"version": "v6.2.0", "version": "v6.2.0",

View file

@ -5,8 +5,10 @@ namespace App\Command;
use App\DataSource\DataSourceException; use App\DataSource\DataSourceException;
use App\DataSource\DataSourceInterface; use App\DataSource\DataSourceInterface;
use App\DataSource\ErinReedDataSource; use App\DataSource\ErinReedDataSource;
use App\DataSource\TransInTheSouthDataSource;
use App\Entity\Clinic; use App\Entity\Clinic;
use App\Entity\ImportHash; use App\Entity\ImportHash;
use App\HereMaps\Client as HereClient;
use App\Repository\ClinicRepository; use App\Repository\ClinicRepository;
use App\Repository\ImportHashRepository; use App\Repository\ImportHashRepository;
use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Attribute\AsCommand;
@ -23,19 +25,23 @@ use Symfony\Contracts\HttpClient\HttpClientInterface;
class ClinicsImportCommand extends Command class ClinicsImportCommand extends Command
{ {
private HttpClientInterface $httpClient; private HttpClientInterface $httpClient;
private HereClient $hereClient;
private ClinicRepository $clinics; private ClinicRepository $clinics;
private ImportHashRepository $imports; private ImportHashRepository $imports;
private array $dataSources = [ private array $dataSources = [
ErinReedDataSource::class ErinReedDataSource::class,
TransInTheSouthDataSource::class,
]; ];
public function __construct( public function __construct(
HttpClientInterface $httpClient, HttpClientInterface $httpClient,
HereClient $hereClient,
ClinicRepository $clinics, ClinicRepository $clinics,
ImportHashRepository $imports, ImportHashRepository $imports,
) { ) {
$this->httpClient = $httpClient; $this->httpClient = $httpClient;
$this->hereClient = $hereClient;
$this->clinics = $clinics; $this->clinics = $clinics;
$this->imports = $imports; $this->imports = $imports;
parent::__construct(); parent::__construct();
@ -49,7 +55,7 @@ class ClinicsImportCommand extends Command
$clinicsAddedCount = 0; $clinicsAddedCount = 0;
foreach ($this->dataSources as $source) { foreach ($this->dataSources as $source) {
/* @var DataSourceInterface $source */ /* @var DataSourceInterface $source */
$source = new $source($this->httpClient); $source = new $source($this->httpClient, $this->hereClient);
$io->section($source->getType()); $io->section($source->getType());
$io->text('Fetching clinics'); $io->text('Fetching clinics');
@ -72,6 +78,7 @@ class ClinicsImportCommand extends Command
continue; continue;
} }
$source->preImport($new);
$this->clinics->save($new); $this->clinics->save($new);
$import = new ImportHash(); $import = new ImportHash();

View file

@ -14,7 +14,7 @@ use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle; use Symfony\Component\Console\Style\SymfonyStyle;
#[AsCommand( #[AsCommand(
name: 'app:reset', name: 'app:clinics:reset',
description: 'Dev tool used to reset the application database', description: 'Dev tool used to reset the application database',
)] )]
class ResetCommand extends Command class ResetCommand extends Command

View file

@ -21,6 +21,7 @@ use EasyCorp\Bundle\EasyAdminBundle\Filter\ChoiceFilter;
use EasyCorp\Bundle\EasyAdminBundle\Router\AdminUrlGenerator; use EasyCorp\Bundle\EasyAdminBundle\Router\AdminUrlGenerator;
use Oefenweb\DamerauLevenshtein\DamerauLevenshtein as Levenshtein; use Oefenweb\DamerauLevenshtein\DamerauLevenshtein as Levenshtein;
use Symfony\Component\HttpFoundation\RedirectResponse; use Symfony\Component\HttpFoundation\RedirectResponse;
use Symfony\Component\Validator\Constraints\Choice;
class ClinicCrudController extends AbstractCrudController class ClinicCrudController extends AbstractCrudController
{ {
@ -76,7 +77,12 @@ class ClinicCrudController extends AbstractCrudController
->add(ChoiceFilter::new('published')->setChoices([ ->add(ChoiceFilter::new('published')->setChoices([
'No' => false, 'No' => false,
'Yes' => true 'Yes' => true
])); ]))
->add(ChoiceFilter::new('dataSource')->canSelectMultiple()->setChoices([
'Trans in the South' => 'transInTheSouth',
'Erin Reed' => 'erinReed',
]))
;
} }
public function configureFields(string $pageName): iterable public function configureFields(string $pageName): iterable

View file

@ -3,16 +3,19 @@
namespace App\DataSource; namespace App\DataSource;
use App\Entity\Clinic; use App\Entity\Clinic;
use App\HereMaps\Client;
use Symfony\Contracts\HttpClient\HttpClientInterface; use Symfony\Contracts\HttpClient\HttpClientInterface;
interface DataSourceInterface interface DataSourceInterface
{ {
public const DATASOURCE__ERIN_REED = 'erinReed'; public const DATASOURCE__ERIN_REED = 'erinReed';
public const DATASOURCE__TRANS_IN_THE_SOUTH = 'transInTheSouth';
public const DATASOURCE__MANUAL_ENTRY = 'manualEntry'; public const DATASOURCE__MANUAL_ENTRY = 'manualEntry';
public function __construct(HttpClientInterface $httpClient); public function __construct(HttpClientInterface $httpClient, Client $hereClient);
public function getType(): string; public function getType(): string;
/* @throws DataSourceException */ /* @throws DataSourceException */
public function fetchClinics(): array; public function fetchClinics(): array;
public function hash(Clinic $clinic): string; public function hash(Clinic $clinic): string;
public function preImport(Clinic $clinic): void;
} }

View file

@ -3,6 +3,7 @@
namespace App\DataSource; namespace App\DataSource;
use App\Entity\Clinic; use App\Entity\Clinic;
use App\HereMaps\Client as HereClient;
use Symfony\Contracts\HttpClient\HttpClientInterface; use Symfony\Contracts\HttpClient\HttpClientInterface;
class ErinReedDataSource implements DataSourceInterface class ErinReedDataSource implements DataSourceInterface
@ -11,8 +12,10 @@ class ErinReedDataSource implements DataSourceInterface
private HttpClientInterface $httpClient; private HttpClientInterface $httpClient;
public function __construct(HttpClientInterface $httpClient) public function __construct(
{ HttpClientInterface $httpClient,
HereClient $hereClient,
) {
$this->httpClient = $httpClient; $this->httpClient = $httpClient;
} }
@ -103,4 +106,9 @@ class ErinReedDataSource implements DataSourceInterface
$data = implode( '.', $pieces ); $data = implode( '.', $pieces );
return md5( $data ); return md5( $data );
} }
public function preImport(Clinic $clinic): void
{
// left intentionally blank
}
} }

View file

@ -0,0 +1,159 @@
<?php
namespace App\DataSource;
use App\Entity\Clinic;
use App\HereMaps\Client as HereClient;
use Symfony\Component\DomCrawler\Crawler;
use Symfony\Contracts\HttpClient\HttpClientInterface;
class TransInTheSouthDataSource implements DataSourceInterface {
private HttpClientInterface $httpClient;
private HereClient $hereClient;
public function __construct(
HttpClientInterface $httpClient,
HereClient $hereClient,
) {
$this->httpClient = $httpClient;
$this->hereClient = $hereClient;
}
public function getType(): string
{
return self::DATASOURCE__TRANS_IN_THE_SOUTH;
}
/**
* @throws DataSourceException
*/
public function fetchClinics(): array
{
$tisProvidersId = $this->_scrapeTisProvidersId();
$html = $this->_loadSearchResults($tisProvidersId);
$rawRecords = $this->_scrapeData($html);
$clinics = [];
foreach ($rawRecords as $record) {
$clinic = new Clinic();
$clinic->setName($record['name']);
$clinic->setDescription($record['description']);
$clinic->setAddress($record['address']);
$clinic->setDataSource($this->getType());
$clinic->setPublished(false);
$clinics[] = $clinic;
}
return $clinics;
}
private function _scrapeTisProvidersId(): string
{
try {
$res = $this->httpClient->request('GET', 'https://southernequality.org/resources/transinthesouth/');
$html = $res->getContent();
} catch (\Throwable) {
throw new DataSourceException('HTTP request to fetch search form failed.', $this->getType());
}
if ($html === '') {
throw new DataSourceException('Missing web page content in response.', $this->getType());
}
$crawler = new Crawler($html);
return $crawler->filter('#filter-tis-providers')->first()->attr('value');
}
/**
* @throws DataSourceException
*/
private function _loadSearchResults(string $tisProvidersId): string
{
try {
$res = $this->httpClient->request('POST', 'https://southernequality.org/resources/transinthesouth/', [
'body' => 'tis-name-search=&states=&services%5B%5D=Informed+Consent&services%5B%5D=Offers+Hormone+Replacement+Therapy+%28HRT%29&filter-tis-providers=' . $tisProvidersId . '&_wp_http_referer=%2Fresources%2Ftransinthesouth%2F&filter_providers=Search',
]);
$html = $res->getContent();
} catch (\Throwable) {
throw new DataSourceException('HTTP request to fetch search results failed.', $this->getType());
}
if ($html === '') {
throw new DataSourceException('Missing web page content in response.', $this->getType());
}
return $html;
}
/**
* @throws DataSourceException
*/
private function _scrapeData(string $html): array
{
$rawRecords = [];
$crawler = new Crawler($html);
$crawler->filter('.provider')->each(function(Crawler $crawler) use (&$rawRecords) {
$services = [];
foreach ($crawler->filter('.accordion-header') as $service) {
$services[] = $service->textContent;
}
if (!in_array('Offers Hormone Replacement Therapy (HRT)', $services) || !in_array('Informed Consent', $services)) {
return;
}
$name = $crawler->filter('.provider--title')->text();
if (!$name) {
throw new DataSourceException('Missing clinic attribute in scraped data: name', $this->getType());
}
$practice = $crawler->filter('.provider--practice-name')->text();
if ($practice && ($name !== $practice)) {
$name .= ' - ' . $practice;
}
$description = $crawler->filter('.provider--summary')->text();
$address = $crawler->filter('.provider--address')->text();
if (!$address) {
throw new DataSourceException('Missing clinic attribute in scraped data: address', $this->getType());
}
$rawRecords[] = [
'name' => $name,
'description' => $description,
'address' => $address,
];
});
return $rawRecords;
}
public function hash(Clinic $clinic): string
{
$pieces = [
$clinic->getName(),
$clinic->getDescription(),
$clinic->getAddress(),
];
$data = implode('.', $pieces);
return md5($data);
}
/**
* @throws \Exception
* @throws DataSourceException
*/
public function preImport(Clinic $clinic): void
{
$items = $this->hereClient->geocode($clinic->getAddress())['items'];
if (count($items) === 0) {
throw new DataSourceException('No coordinates found for address: ' . $clinic->getAddress(), $this->getType());
}
$clinic->setLatitude($items[0]['position']['lat']);
$clinic->setLongitude($items[0]['position']['lng']);
}
}

View file

@ -46,4 +46,30 @@ class Client {
return $decoded; return $decoded;
} }
/**
* @throws \Exception
*/
public function geocode(string $address): array
{
$url = 'https://geocode.search.hereapi.com/v1/geocode?' . http_build_query([
'lang' => 'en-US',
'q' => $address,
'apiKey' => $this->hereApiKey,
]);
try {
$res = $this->httpClient->request('GET', $url);
$data = $res->getContent();
} catch (\Throwable $e) {
throw new \Exception('HTTP request to Here Maps failed: ' . $e->getMessage());
}
$decoded = json_decode($data, true);
if ($decoded === false) {
throw new \Exception('Failed to decode Here Maps response');
}
return $decoded;
}
} }

View file

@ -24,6 +24,7 @@ Parameters:
<div class="d-grid"> <div class="d-grid">
{{ form_widget(searchForm.submit, {'attr': {'class': 'rounded-pill btn-primary'}}) }} {{ form_widget(searchForm.submit, {'attr': {'class': 'rounded-pill btn-primary'}}) }}
</div> </div>
{{ form_widget(searchForm.page, {'attr': {'value': '1'}}) }}
</div> </div>
{{ form_end(searchForm) }} {{ form_end(searchForm) }}
</div> </div>

View file

@ -49,12 +49,20 @@
{% if clinic.address %} {% if clinic.address %}
<p class="mb-2">{{ clinic.address }}</p> <p class="mb-2">{{ clinic.address }}</p>
{% endif %} {% endif %}
{% if clinic.dataSource != 'manualEntry' %}
<span class="badge text-bg-secondary me-2">
<i class="fa-solid fa-database"></i>
{% if clinic.dataSource == "erinReed" %} {% if clinic.dataSource == "erinReed" %}
<span class="badge bg-blue text-dark-blue" style="color: rgb(0, 77, 124);"> <a class="text-white" target="_blank" href="https://www.google.com/maps/d/viewer?mid=1DxyOTw8dI8n96BHFF2JVUMK7bXsRKtzA&ll=41.639103490264155%2C-83.26303679041781&z=3">Erin Reed</a>
{% elseif clinic.dataSource == 'transInTheSouth' %}
<a class="text-white" target="_blank" href="https://southernequality.org/resources/transinthesouth/">Trans in the South</a>
{% endif %}
</span>
{% endif %}
<span class="badge bg-blue text-dark-blue">
<i class="fa-solid fa-check"></i> <i class="fa-solid fa-check"></i>
Informed Consent Informed Consent
</span> </span>
{% endif %}
{% if clinic.description %} {% if clinic.description %}
<hr> <hr>
<p class="mb-0">{{ clinic.description|raw }}</p> <p class="mb-0">{{ clinic.description|raw }}</p>