Commit d988c813 authored by Sebastian Castro's avatar Sebastian Castro

Import Mapping: Ability to run custom code

parent 997c4138
......@@ -29,7 +29,7 @@ class AppExtension extends AbstractExtension
public function values($value)
{
return array_values($value);
return $value ? array_values($value) : [];
}
public function getFunctions()
......
......@@ -46,7 +46,7 @@ class MigrationCommand extends GoGoAbstractCommand
}
protected function gogoExecute($em, InputInterface $input, OutputInterface $output)
{
{
$migrationState = $em->createQueryBuilder('BiopenCoreBundle:MigrationState')->getQuery()->getSingleResult();
if ($migrationState == null) // Meaning the migration state was not yet in the place in the code
{
......@@ -57,7 +57,7 @@ class MigrationCommand extends GoGoAbstractCommand
try {
// Collecting the Database to be updated
$dbs = ['gogocarto_default'];
$dbNames = $em->createQueryBuilder('BiopenSaasBundle:Project')->select('domainName')->hydrate(false)->getQuery()->execute()->toArray();
$dbNames = $em->createQueryBuilder('BiopenSaasBundle:Project')->select('domainName')->hydrate(false)->getQuery()->execute()->toArray();
foreach ($dbNames as $object) { $dbs[] = $object['domainName']; }
if (count($this->migrations) > $migrationState->getMigrationIndex()) {
......@@ -65,7 +65,7 @@ class MigrationCommand extends GoGoAbstractCommand
foreach($dbs as $db) {
foreach($migrationsToRun as $migration) {
$this->runCommand($db, $migration);
}
}
}
$this->log(count($migrationsToRun) . " migrations performed");
} else {
......@@ -78,9 +78,9 @@ class MigrationCommand extends GoGoAbstractCommand
if (count($this->commands) > $migrationState->getCommandsIndex()) {
$commandsToRun = array_slice($this->commands, $migrationState->getCommandsIndex());
foreach($dbs as $db) {
foreach($commandsToRun as $command) {
foreach($commandsToRun as $command) {
$asyncService->callCommand($command, [], $db);
}
}
}
$this->log(count($commandsToRun) . " commands to run");
} else {
......@@ -91,24 +91,24 @@ class MigrationCommand extends GoGoAbstractCommand
$messagesToAdd = array_slice($this->messages, $migrationState->getMessagesIndex());
foreach($dbs as $db) {
foreach($messagesToAdd as $message) {
// create a GoGoLogUpdate
// create a GoGoLogUpdate
$asyncService->callCommand('gogolog:add:message', ['"' . $message . '"'], $db);
}
}
}
$this->log(count($messagesToAdd) . " messages added to admin dashboard");
} else {
$this->log("No Messages to add to dashboard");
}
}
catch (\Exception $e) {
$message = $e->getMessage() . '</br>' . $e->getFile() . ' LINE ' . $e->getLine();
catch (\Exception $e) {
$message = $e->getMessage() . '</br>' . $e->getFile() . ' LINE ' . $e->getLine();
$this->error("Error performing migrations: " . $message);
}
}
$migrationState->setMigrationIndex(count($this->migrations));
$migrationState->setCommandsIndex(count($this->commands));
$migrationState->setMessagesIndex(count($this->messages));
$em->flush();
$em->flush();
}
private function runCommand($db, $command)
......
......@@ -57,7 +57,23 @@ class ImportDynamicAdmin extends ImportAbstractAdmin
->add('logs', 'hidden', array('attr' => ['class' => 'gogo-display-logs'], 'mapped' => false))
->end();
}
$formMapper->end();
$formMapper->end();
$formMapper->tab('Modifier les données en exécutant du code')
->with('Entrez du code qui sera exécuté à la reception des données, avant leur traitement par GoGoCarto', ["description" => "La variable <b>\$data</b> représente le tableau PHP créé à partir des données Csv ou Json. Quelques examples de transformations simple:
<pre>&lt;?php</br>\$data = \$data['elements']</pre>
<pre>&lt;?php</br>foreach(\$data as \$key => \$row) {
\$data[\$key]['source'] = \"MySource\";
}</pre>
<pre>&lt;?php</br>foreach(\$data as \$key => \$row) {
\$data[\$key]['latitude'] = \$row['geo']['latitude']);
\$data[\$key]['longitude'] = \$row['geo']['longitude']);
}</pre>
<pre>&lt;?php</br>foreach(\$data as \$key => \$row) {
\$data[\$key]['categories'] = array_map(function(\$cat) { return \$cat[0]; }, \$row['categories']);
}</pre>"])
->add('customCode', 'text', array('label' => 'Code PHP qui sera exécuté', 'attr' => ['class' => 'gogo-code-editor', 'format' => 'php', 'height' => '500'], 'required' => false))
->end()
->end();
if ($this->getSubject()->getId())
{
......
......@@ -7,6 +7,9 @@ use Symfony\Component\HttpFoundation\File\File;
use Vich\UploaderBundle\Mapping\Annotation as Vich;
use Biopen\CoreBundle\Document\AbstractFile;
use Symfony\Component\Validator\Constraints as Assert;
use Symfony\Component\Validator\Context\ExecutionContextInterface;
abstract class ImportState
{
const Started = "started";
......@@ -119,6 +122,12 @@ class Import extends AbstractFile
*/
private $taxonomyMapping = [];
/**
* Custom code made by the user to be run on the $data object when importing
* @MongoDB\Field(type="string")
*/
private $customCode = "<?php";
public function __construct() {
$this->logs = new \Doctrine\Common\Collections\ArrayCollection();;
......@@ -135,7 +144,19 @@ class Import extends AbstractFile
public function isCategoriesFieldMapped()
{
return in_array('categories', array_values($this->getOntologyMapping()));
return $this->getOntologyMapping() ? in_array('categories', array_values($this->getOntologyMapping())) : false;
}
/**
* @Assert\Callback
*/
public function validate(ExecutionContextInterface $context)
{
if (preg_match("/new |process|mongo/i", $this->customCode)) {
$context->buildViolation("Il est interdit d'utiliser les mots suivants: new, mongo, process... Merci de ne pas faire de betises !")
->atPath('customCode')
->addViolation();
}
}
/**
......@@ -473,4 +494,26 @@ class Import extends AbstractFile
{
return $this->taxonomyMapping;
}
/**
* Set customCode
*
* @param string $customCode
* @return $this
*/
public function setCustomCode($customCode)
{
$this->customCode = $customCode;
return $this;
}
/**
* Get customCode
*
* @return string $customCode
*/
public function getCustomCode()
{
return $this->customCode;
}
}
......@@ -49,6 +49,35 @@ class ElementImportMappingService
$parent = $import->getParentCategoryToCreateOptions() ?: $this->em->getRepository('BiopenGeoDirectoryBundle:Category')->findOneByIsRootCategory(true);
$this->parentCategoryIdToCreateMissingOptions = $parent->getId();
// elements is ofently stored nested in a data attribute
if (array_key_exists('data', $data)) $data = $data['data'];
// Fixs gogocarto ontology when importing
foreach ($data as $key => $row) {
if (array_key_exists('geo', $row))
{
$data[$key]['latitude'] = $row['geo']['latitude'];
$data[$key]['longitude'] = $row['geo']['longitude'];
unset($data[$key]['geo']);
}
if (array_key_exists('address', $row))
{
$address = $row['address'];
if (gettype($address) == "string") $data[$key]['streetAddress'] = $address;
else if ($address) {
if (array_key_exists('streetAddress', $address)) $data[$key]['streetAddress'] = $address['streetAddress'];
if (array_key_exists('addressLocality', $address)) $data[$key]['addressLocality'] = $address['addressLocality'];
if (array_key_exists('postalCode', $address)) $data[$key]['postalCode'] = $address['postalCode'];
if (array_key_exists('addressCountry', $address)) $data[$key]['addressCountry'] = $address['addressCountry'];
}
unset($data[$key]['address']);
}
}
// Execute custom code (the <?php is used to have proper code highliting in text editor, we remove it before executing)
eval(str_replace('<?php', '', $import->getCustomCode()));
$this->collectOntology($data, $import);
$data = $this->mapOntology($data);
......@@ -163,11 +192,13 @@ class ElementImportMappingService
$mapping = $this->import->getTaxonomyMapping();
foreach ($data as $key => $row)
{
if (is_string($row['categories'])) $row['categories'] = explode(',', $row['categories']);
$categories = array_map(function($el) use ($mapping) {
return array_key_exists($el, $mapping) ? $this->mappingTableIds[$mapping[$el]]['idAndParentsId'] : [];
}, $row['categories']);
$data[$key]['categories'] = array_unique($this->array_flatten($categories));
if (is_array($row['categories'])) {
if (is_string($row['categories'])) $row['categories'] = explode(',', $row['categories']);
$categories = array_map(function($el) use ($mapping) {
return array_key_exists($el, $mapping) && array_key_exists($mapping[$el], $this->mappingTableIds) ? $this->mappingTableIds[$mapping[$el]]['idAndParentsId'] : [];
}, $row['categories']);
$data[$key]['categories'] = array_unique($this->array_flatten($categories));
}
}
return $data;
}
......
......@@ -89,31 +89,6 @@ class ElementImportService
$data = json_decode($json, true);
if ($data === null) return null;
// data can be stored inside a data attribute
if (array_key_exists('data', $data)) $data = $data['data'];
foreach ($data as $key => $row) {
if (array_key_exists('geo', $row))
{
$data[$key]['latitude'] = $row['geo']['latitude'];
$data[$key]['longitude'] = $row['geo']['longitude'];
unset($data[$key]['geo']);
}
if (array_key_exists('address', $row))
{
$address = $row['address'];
if (gettype($address) == "string") $data[$key]['streetAddress'] = $address;
else if ($address) {
if (array_key_exists('streetAddress', $address)) $data[$key]['streetAddress'] = $address['streetAddress'];
if (array_key_exists('addressLocality', $address)) $data[$key]['addressLocality'] = $address['addressLocality'];
if (array_key_exists('postalCode', $address)) $data[$key]['postalCode'] = $address['postalCode'];
if (array_key_exists('addressCountry', $address)) $data[$key]['addressCountry'] = $address['addressCountry'];
}
unset($data[$key]['address']);
}
}
if ($onlyGetData) return $data;
$elementImportedCount = $this->importData($data, $import);
......@@ -125,7 +100,7 @@ class ElementImportService
// create a mapping table for ontology and taxonomy
public function collectData($import)
{
$data = $import->getUrl() ?$this->importJson($import, true) : $this->importCsv($import, true);
$data = $import->getUrl() ? $this->importJson($import, true) : $this->importCsv($import, true);
$this->mappingService->transform($data, $import);
}
......
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed.
{
"licence": "https://opendatacommons.org/licenses/odbl/summary/",
"ontology":"gogofull",
"licence": "https://opendatacommons.org/licenses/odbl/summary/",
"ontology":"gogofull",
"data":[{
"id":"Rhm",
"newfields": "balam",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment