Skip to content

GitLab

G
GoGoCarto
Commit 5949d484 authored by Sebastian Castro's avatar Sebastian Castro
Browse files
Options

Fixes and improvements re importing data

parent 5d957cd6
Hide whitespace changes
Inline Side-by-side
Showing with 122 additions and 44 deletions
src/Biopen/GeoDirectoryBundle/Admin/ImportDynamicAdmin.php
View file @ 5949d484
......@@ -38,8 +38,8 @@ class ImportDynamicAdmin extends AbstractAdmin
->add('refreshFrequencyInDays', null, array('required' => false, 'label' => "Fréquence de mise à jours des données en jours (laisser vide pour ne jamais mettre à jour automatiquement"))
->add('idsToIgnore', 'text', array('required' => false, 'attr' => ['class' => 'gogo-display-array'], 'label' => "Liste des IDs à ignorer lors de l'import (pour ignorer un élément, supprimer le et il ne sera plus jamais importé"))
->end()
->with('Hitorique', array('class' => 'col-sm-12'))
->add('logs', 'hidden', array('attr' => ['class' => 'gogo-display-logs']))
->with('Historique', array('class' => 'col-sm-12'))
->add('logs', 'hidden', array('attr' => ['class' => 'gogo-display-logs'], 'mapped' => false))
->end();
}
......
src/Biopen/GeoDirectoryBundle/Block/MonitoringElementsBlockService.php
View file @ 5949d484
......@@ -55,6 +55,7 @@ class MonitoringElementsBlockService extends AbstractBlockService
$activeUsersNewsletterCount = $this->em->createQueryBuilder('BiopenCoreBundle:User')->field('enabled')->equals(true)
->field('newsletterFrequency')->gt(NewsletterFrequencyOptions::Never)->count()->getQuery()->execute();
$errors = $this->em->getRepository('BiopenCoreBundle:GoGoLog')->findBy(['type' => 'error', 'hidden' => false]);
usort( $errors, function ($a, $b) { return $b->getCreatedAt()->getTimestamp() - $a->getCreatedAt()->getTimestamp(); });
// merge settings
$settings = $blockContext->getSettings();
......
src/Biopen/GeoDirectoryBundle/Command/ImportSourceCommand.php
View file @ 5949d484
......@@ -27,7 +27,6 @@ class ImportSourceCommand extends GoGoAbstractCommand
$sourceNameOrId = $input->getArgument('sourceNameOrImportId');
$import = $em->getRepository('BiopenGeoDirectoryBundle:ImportDynamic')->find($sourceNameOrId);
if (!$import) $import = $em->getRepository('BiopenGeoDirectoryBundle:ImportDynamic')->findOneBySourceName($sourceNameOrId);
if (!$import)
{
$message = "ERREUR pendant l'import : Aucune source avec pour nom ou id " . $input->getArgument('sourceNameOrImportId') . " n'existe dans la base de donnée " . $input->getArgument('dbname');
......@@ -39,9 +38,10 @@ class ImportSourceCommand extends GoGoAbstractCommand
$importService = $this->getContainer()->get('biopen.element_import');
$dataToImport = $importService->importJson($import, true);
$this->log('Data downloaded. ' . count($dataToImport) . ' elements to import...');
$this->log($importService->importData($dataToImport, $import));
$result = $importService->importData($dataToImport, $import, $this->logger);
$this->log($result);
} catch (\Exception $e) {
$this->error($e->getMessage());
$this->error("Source: " . $import->getSourceName() . " - " . $e->getMessage());
}
}
......
src/Biopen/GeoDirectoryBundle/Controller/Admin/ImportDynamicAdminController.php
View file @ 5949d484
......@@ -18,6 +18,6 @@ class ImportDynamicAdminController extends Controller
// $result = $this->get('biopen.element_import')->importJson($object);
// $this->addFlash('sonata_flash_success', $result);
return $this->redirect($this->admin->generateUrl('list'));
return $this->redirect($this->admin->generateUrl('edit', ['id' => $object->getId()]));
}
}
\ No newline at end of file
src/Biopen/GeoDirectoryBundle/Document/Element.php
View file @ 5949d484
......@@ -316,7 +316,9 @@ class Element
* so noone else make action on the same element
* @MongoDB\Field(type="int")
*/
private $lockUntil = 0;
private $lockUntil = 0;
private $preventJsonUpdate = false;
/**
* Constructor
......@@ -447,8 +449,10 @@ class Element
/** @MongoDB\PreFlush */
public function onPreFlush()
{
$this->checkForModerationNeeded();
$this->updateJsonRepresentation();
if (!$this->getPreventJsonUpdate()) {
$this->checkForModerationNeeded();
$this->updateJsonRepresentation();
}
}
// automatically resolve moderation error
......@@ -498,9 +502,9 @@ class Element
if ($this->openHours) $baseJson .= ', "openHours": ' . $this->openHours->toJson();
// CREATED AT, UPDATED AT
$baseJson .= ', "createdAt":"' . date_format($this->createdAt,"d/m/Y") . '"';
$baseJson .= ', "createdAt":"' . date_format($this->createdAt,"d/m/Y à H:i") . '"';
$updatedAt = $this->updatedAt ? $this->updatedAt : $this->createdAt;
$updatedAtFormated = gettype($updatedAt) == "integer" ? date("d/m/Y", $updatedAt) : date_format($updatedAt,"d/m/Y");
$updatedAtFormated = gettype($updatedAt) == "integer" ? date("d/m/Y à H:i", $updatedAt) : date_format($updatedAt,"d/m/Y à H:i");
$baseJson .= ', "updatedAt":"' . $updatedAtFormated . '"';
// OPTIONS VALUES (= TAXONOMY)
......@@ -527,9 +531,10 @@ class Element
if (count($optionDescriptionsJson)) $baseJson .= '"categoriesDescriptions": [' . implode(",", $optionDescriptionsJson) . '],';
// CUSTOM DATA
foreach ($this->getData() as $key => $value) {
$baseJson .= '"'. $key .'": ' . json_encode($value) . ',';
}
if ($this->getData())
foreach ($this->getData() as $key => $value) {
$baseJson .= '"'. $key .'": ' . json_encode($value) . ',';
}
// SPECIFIC DATA
$baseJson .= $this->encodeArrayObjectToJson("stamps", $this->stamps);
......@@ -708,16 +713,20 @@ class Element
public function setCustomData($data, $privateProps)
{
if (array_key_exists('email', $data)) {
$this->setEmail($data['email']);
}
$privateData = [];
foreach ($privateProps as $key => $prop) {
if (array_key_exists($prop, $data)) {
$privateData[$prop] = $data[$prop];
unset($data[$prop]);
if ($data != null)
{
if (array_key_exists('email', $data)) {
$this->setEmail($data['email']);
}
}
foreach ($privateProps as $key => $prop) {
if (array_key_exists($prop, $data)) {
$privateData[$prop] = $data[$prop];
unset($data[$prop]);
}
}
}
if ($this->getData()) $data = array_merge($this->getData(), $data); // keeping also old data
$this->setData($data);
......@@ -1526,8 +1535,17 @@ class Element
public function getEmail()
{
if ($this->email) return $this->email;
if (array_key_exists('email', $this->data)) return $this->data['email'];
if (array_key_exists('email', $this->privateData)) return $this->privateData['email'];
if ($this->data && array_key_exists('email', $this->data)) return $this->data['email'];
if ($this->privateData && array_key_exists('email', $this->privateData)) return $this->privateData['email'];
return "";
}
public function setPreventJsonUpdate($preventJsonUpdate)
{
$this->preventJsonUpdate = $preventJsonUpdate;
return $this;
}
public function getPreventJsonUpdate() { return $this->preventJsonUpdate || false;}
}
src/Biopen/GeoDirectoryBundle/Document/Import.php
View file @ 5949d484
......@@ -65,7 +65,9 @@ class Import extends AbstractFile
*/
private $logs;
public function __construct() {}
public function __construct() {
$this->logs = new \Doctrine\Common\Collections\ArrayCollection();;
}
public function __toString() { return "Import " . $this->sourceName; }
......@@ -247,7 +249,7 @@ class Import extends AbstractFile
* @return \Doctrine\Common\Collections\Collection $logs
*/
public function getLogs()
{
{
$logs = is_array($this->logs) ? $this->logs : $this->logs->toArray();
usort( $logs, function ($a, $b) { return $b->getCreatedAt()->getTimestamp() - $a->getCreatedAt()->getTimestamp(); });
return $logs;
......
src/Biopen/GeoDirectoryBundle/Resources/views/admin/core_custom/custom-fields/display-logs.html.twig
View file @ 5949d484
......@@ -19,7 +19,7 @@
{% if dismissLog is defined %}
<button type="button" class="close" data-dismiss="alert" aria-label="Fermer" onclick="hideLog({{log.id}})"><span aria-hidden="true" style="font-size: 1.6rem">Ne plus afficher</span></button>
{% endif %}
<strong>{{ log.createdAt|date("d/m/Y à H:i") }}</strong> - {{ log.message }}
<strong>{{ log.createdAt|date("d/m/Y à H:i") }}</strong> - {{ log.message|raw }}
</div>
{% endfor %}
......
src/Biopen/GeoDirectoryBundle/Services/ElementImportService.php
View file @ 5949d484
......@@ -40,6 +40,7 @@ class ElementImportService
protected $countElementUpdated = 0;
protected $countElementNothingToDo = 0;
protected $countElementErrors = 0;
protected $elementIdsErrors = [];
/**
* Constructor
*/
......@@ -84,7 +85,7 @@ class ElementImportService
$data[$key]['longitude'] = $row['geo']['longitude'];
unset($data[$key]['geo']);
}
if (array_key_exists('address', $row))
if (array_key_exists('address', $row))
{
$address = $row['address'];
......@@ -139,15 +140,26 @@ class ElementImportService
$data = $this->fixsOntology($data);
$data = $this->addMissingFieldsToData($data);
if ($import->isDynamicImport())
{
// before updating the source, we put all elements into DynamicImportTemp status
$qb = $this->em->createQueryBuilder('BiopenGeoDirectoryBundle:Element');
$qb->updateMany()
->field('source')->references($import)
->field('status')->set(ElementStatus::DynamicImportTemp)
->getQuery()->execute();
}
// processing each data
foreach($data as $element)
foreach($data as $row)
{
try {
$this->createElementFromArray($element, $import);
try {
$this->createElementFromArray($row, $import);
$i++;
}
catch (\Exception $e) {
$this->countElementErrors++;
$this->elementIdsErrors[] = "" . $row['id'];
}
if (($i % $batchSize) === 0)
......@@ -158,34 +170,79 @@ class ElementImportService
$import = $this->em->getRepository('BiopenGeoDirectoryBundle:ImportDynamic')->find($import->getId());
}
}
$result = "Import terminé";
if ($this->countElementCreated > 0) $result .= ", " . $this->countElementCreated . " éléments importés";
if ($this->countElementUpdated > 0) $result .= ", " . $this->countElementUpdated . " élements mis à jours";
if ($this->countElementNothingToDo > 0) $result .= ", " . $this->countElementNothingToDo . " élements laissés tels quels (rien à mettre à jour)";
if ($this->countElementErrors > 0) $result .= ", " . $this->countElementErrors . " erreurs pendant l'import";
$logType = $this->countElementErrors > 0 ? ($this->countElementErrors > ($size / 4) ? 'error' : 'warning') : 'success';
$log = new GoGoLog($logType, $result);
$import->addLog($log);
$this->em->flush();
$this->em->clear();
$import = $this->em->getRepository('BiopenGeoDirectoryBundle:ImportDynamic')->find($import->getId());
$countElemenDeleted = 0;
if ($import->isDynamicImport())
{
if (count($this->elementIdsErrors) > 0)
{
// If there was an error whil retrieving an already existing element
// we set back the status to DynamicImport otherwise it will be deleted just after
$qb = $this->em->createQueryBuilder('BiopenGeoDirectoryBundle:Element');
$result = $qb->updateMany()
->field('source')->references($import)->field('oldId')->in($this->elementIdsErrors)
->field('status')->set(ElementStatus::DynamicImport)
->getQuery()->execute();
}
// after updating the source, the element still in DynamicImportTemp are the one who are missing
// from the new data received, so we need to delete them
$qb = $this->em->createQueryBuilder('BiopenGeoDirectoryBundle:Element');
$result = $qb->remove()
->field('source')->references($import)
->field('status')->equals(ElementStatus::DynamicImportTemp)
->getQuery()->execute();
$countElemenDeleted = $result['n'];
}
$qb = $this->em->createQueryBuilder('BiopenGeoDirectoryBundle:Element');
$totalCount = $qb->field('status')->field('source')->references($import)->count()->getQuery()->execute();
$result = "Import de " . $import->getSourceName() . " terminé - <strong>Total: " . $totalCount . "</strong>";
if ($this->countElementCreated > 0) $result .= " - " . $this->countElementCreated . " élément.s importé.s";
if ($this->countElementUpdated > 0) $result .= " - " . $this->countElementUpdated . " élement.s mis à jour";
if ($this->countElementNothingToDo > 0) $result .= " - " . $this->countElementNothingToDo . " élement.s laissé.s tel.s quel.s (rien à mettre à jour)";
if ($countElemenDeleted > 0) $result .= " - " . $countElemenDeleted . " élement.s supprimé.s";
if ($this->countElementErrors > 0) $result .= " - " . $this->countElementErrors . " erreur.s pendant l'import";
$logType = $this->countElementErrors > 0 ? ($this->countElementErrors > ($size / 4) ? 'error' : 'warning') : 'success';
$log = new GoGoLog($logType, $result);
$import->addLog($log);
$this->em->flush();
return $result;
}
private function createElementFromArray($row, $import)
{
if (in_array($row['id'], $import->getIdsToIgnore())) return;
$qb = $this->em->createQueryBuilder('BiopenGeoDirectoryBundle:Element');
$qb->field('source')->references($import);
$qb->field('oldId')->equals("" . $row['id']);
$element = $qb->getQuery()->getSingleResult();
if ($row['id'])
{
if (in_array($row['id'], $import->getIdsToIgnore())) return;
$qb = $this->em->createQueryBuilder('BiopenGeoDirectoryBundle:Element');
$qb->field('source')->references($import);
$qb->field('oldId')->equals("" . $row['id']);
$element = $qb->getQuery()->getSingleResult();
} else {
$qb = $this->em->createQueryBuilder('BiopenGeoDirectoryBundle:Element');
$qb->field('source')->references($import);
$qb->field('name')->equals($row['name']);
$qb->field('geo.latitude')->equals((float) number_format((float)$row['latitude'], 5));
$qb->field('geo.longitude')->equals((float) number_format((float)$row['longitude'], 5));
$element = $qb->getQuery()->getSingleResult();
}
if ($element) // if element with this Id already exists
{
// if updated date hasn't change, nothing to do
if (array_key_exists('updatedAt', $row) && $row['updatedAt'] == $element->getCustomProperty('updatedAt')) {
$this->countElementNothingToDo++;
$element->setPreventJsonUpdate(true);
$element->setStatus(ElementStatus::DynamicImport);
$this->em->persist($element);
return;
}
$this->countElementUpdated++;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment