Import Mapping: Ability to run custom code

src/Application/Twig/AppExtension.php

@@ -29,7 +29,7 @@ class AppExtension extends AbstractExtension
 
     public function values($value)
     {
-        return array_values($value);
+        return $value ? array_values($value) : [];
     }
 
     public function getFunctions()

src/Biopen/CoreBundle/Command/MigrationCommand.php

@@ -46,7 +46,7 @@ class MigrationCommand extends GoGoAbstractCommand
     }
 
     protected function gogoExecute($em, InputInterface $input, OutputInterface $output)
-    {       
+    {
         $migrationState = $em->createQueryBuilder('BiopenCoreBundle:MigrationState')->getQuery()->getSingleResult();
         if ($migrationState == null) // Meaning the migration state was not yet in the place in the code
         {
@@ -57,7 +57,7 @@ class MigrationCommand extends GoGoAbstractCommand
         try {
             // Collecting the Database to be updated
             $dbs = ['gogocarto_default'];
-            $dbNames = $em->createQueryBuilder('BiopenSaasBundle:Project')->select('domainName')->hydrate(false)->getQuery()->execute()->toArray();            
+            $dbNames = $em->createQueryBuilder('BiopenSaasBundle:Project')->select('domainName')->hydrate(false)->getQuery()->execute()->toArray();
             foreach ($dbNames as $object) { $dbs[] = $object['domainName']; }
 
             if (count($this->migrations) > $migrationState->getMigrationIndex()) {
@@ -65,7 +65,7 @@ class MigrationCommand extends GoGoAbstractCommand
                 foreach($dbs as $db) {
                     foreach($migrationsToRun as $migration) {
                         $this->runCommand($db, $migration);
-                    }                    
+                    }
                 }
                 $this->log(count($migrationsToRun) . " migrations performed");
             } else {
@@ -78,9 +78,9 @@ class MigrationCommand extends GoGoAbstractCommand
             if (count($this->commands) > $migrationState->getCommandsIndex()) {
                 $commandsToRun = array_slice($this->commands, $migrationState->getCommandsIndex());
                 foreach($dbs as $db) {
-                    foreach($commandsToRun as $command) {            
+                    foreach($commandsToRun as $command) {
                         $asyncService->callCommand($command, [], $db);
-                    }                    
+                    }
                 }
                 $this->log(count($commandsToRun) . " commands to run");
             } else {
@@ -91,24 +91,24 @@ class MigrationCommand extends GoGoAbstractCommand
                 $messagesToAdd = array_slice($this->messages, $migrationState->getMessagesIndex());
                 foreach($dbs as $db) {
                     foreach($messagesToAdd as $message) {
-                        // create a GoGoLogUpdate                    
+                        // create a GoGoLogUpdate
                         $asyncService->callCommand('gogolog:add:message', ['"' . $message . '"'], $db);
-                    }                    
+                    }
                 }
                 $this->log(count($messagesToAdd) . " messages added to admin dashboard");
             } else {
                 $this->log("No Messages to add to dashboard");
             }
         }
-        catch (\Exception $e) {   
-            $message = $e->getMessage() . '</br>' . $e->getFile() . ' LINE ' . $e->getLine();      
+        catch (\Exception $e) {
+            $message = $e->getMessage() . '</br>' . $e->getFile() . ' LINE ' . $e->getLine();
             $this->error("Error performing migrations: " . $message);
-        }      
+        }
 
         $migrationState->setMigrationIndex(count($this->migrations));
         $migrationState->setCommandsIndex(count($this->commands));
         $migrationState->setMessagesIndex(count($this->messages));
-        $em->flush();        
+        $em->flush();
     }
 
     private function runCommand($db, $command)

src/Biopen/GeoDirectoryBundle/Admin/ImportDynamicAdmin.php

@@ -57,7 +57,23 @@ class ImportDynamicAdmin extends ImportAbstractAdmin
                         ->add('logs', 'hidden', array('attr' => ['class' => 'gogo-display-logs'], 'mapped' => false))
                     ->end();
                 }
-            $formMapper->end();
+        $formMapper->end();
+        $formMapper->tab('Modifier les données en exécutant du code')
+            ->with('Entrez du code qui sera exécuté à la reception des données, avant leur traitement par GoGoCarto', ["description" => "La variable <b>\$data</b> représente le tableau PHP créé à partir des données Csv ou Json. Quelques examples de transformations simple:
+<pre>&lt;?php</br>\$data = \$data['elements']</pre>
+<pre>&lt;?php</br>foreach(\$data as \$key => \$row) {
+    \$data[\$key]['source'] = \"MySource\";
+}</pre>
+<pre>&lt;?php</br>foreach(\$data as \$key => \$row) {
+    \$data[\$key]['latitude'] = \$row['geo']['latitude']);
+    \$data[\$key]['longitude'] = \$row['geo']['longitude']);
+}</pre>
+<pre>&lt;?php</br>foreach(\$data as \$key => \$row) {
+    \$data[\$key]['categories'] = array_map(function(\$cat) { return \$cat[0]; }, \$row['categories']);
+}</pre>"])
+                ->add('customCode', 'text', array('label' => 'Code PHP qui sera exécuté', 'attr' => ['class' => 'gogo-code-editor', 'format' => 'php', 'height' => '500'], 'required' => false))
+            ->end()
+        ->end();
 
         if ($this->getSubject()->getId())
         {

src/Biopen/GeoDirectoryBundle/Document/Import.php

@@ -7,6 +7,9 @@ use Symfony\Component\HttpFoundation\File\File;
 use Vich\UploaderBundle\Mapping\Annotation as Vich;
 use Biopen\CoreBundle\Document\AbstractFile;
 
+use Symfony\Component\Validator\Constraints as Assert;
+use Symfony\Component\Validator\Context\ExecutionContextInterface;
+
 abstract class ImportState
 {
     const Started = "started";
@@ -119,6 +122,12 @@ class Import extends AbstractFile
      */
     private $taxonomyMapping = [];
 
+    /**
+     * Custom code made by the user to be run on the $data object when importing
+     * @MongoDB\Field(type="string")
+     */
+    private $customCode = "<?php";
+
 
     public function __construct() {
         $this->logs = new \Doctrine\Common\Collections\ArrayCollection();;
@@ -135,7 +144,19 @@ class Import extends AbstractFile
 
     public function isCategoriesFieldMapped()
     {
-        return in_array('categories', array_values($this->getOntologyMapping()));
+        return $this->getOntologyMapping() ? in_array('categories', array_values($this->getOntologyMapping())) : false;
+    }
+
+    /**
+     * @Assert\Callback
+     */
+    public function validate(ExecutionContextInterface $context)
+    {
+        if (preg_match("/new |process|mongo/i", $this->customCode)) {
+            $context->buildViolation("Il est interdit d'utiliser les mots suivants: new, mongo, process... Merci de ne pas faire de betises !")
+                ->atPath('customCode')
+                ->addViolation();
+        }
     }
 
     /**
@@ -473,4 +494,26 @@ class Import extends AbstractFile
     {
         return $this->taxonomyMapping;
     }
+
+    /**
+     * Set customCode
+     *
+     * @param string $customCode
+     * @return $this
+     */
+    public function setCustomCode($customCode)
+    {
+        $this->customCode = $customCode;
+        return $this;
+    }
+
+    /**
+     * Get customCode
+     *
+     * @return string $customCode
+     */
+    public function getCustomCode()
+    {
+        return $this->customCode;
+    }
 }

src/Biopen/GeoDirectoryBundle/Services/ElementImportMappingService.php

@@ -49,6 +49,35 @@ class ElementImportMappingService
     $parent = $import->getParentCategoryToCreateOptions() ?: $this->em->getRepository('BiopenGeoDirectoryBundle:Category')->findOneByIsRootCategory(true);
     $this->parentCategoryIdToCreateMissingOptions = $parent->getId();
 
+    // elements is ofently stored nested in a data attribute
+    if (array_key_exists('data', $data)) $data = $data['data'];
+
+    // Fixs gogocarto ontology when importing
+    foreach ($data as $key => $row) {
+      if (array_key_exists('geo', $row))
+      {
+        $data[$key]['latitude']  = $row['geo']['latitude'];
+        $data[$key]['longitude'] = $row['geo']['longitude'];
+        unset($data[$key]['geo']);
+      }
+      if (array_key_exists('address', $row))
+      {
+        $address = $row['address'];
+
+        if (gettype($address) == "string") $data[$key]['streetAddress'] = $address;
+        else if ($address) {
+          if (array_key_exists('streetAddress', $address))   $data[$key]['streetAddress']   = $address['streetAddress'];
+          if (array_key_exists('addressLocality', $address)) $data[$key]['addressLocality'] = $address['addressLocality'];
+          if (array_key_exists('postalCode', $address))      $data[$key]['postalCode']      = $address['postalCode'];
+          if (array_key_exists('addressCountry', $address))  $data[$key]['addressCountry']  = $address['addressCountry'];
+        }
+        unset($data[$key]['address']);
+      }
+    }
+
+    // Execute custom code (the <?php is used to have proper code highliting in text editor, we remove it before executing)
+    eval(str_replace('<?php', '', $import->getCustomCode()));
+
     $this->collectOntology($data, $import);
     $data = $this->mapOntology($data);
 
@@ -163,11 +192,13 @@ class ElementImportMappingService
     $mapping = $this->import->getTaxonomyMapping();
     foreach ($data as $key => $row)
     {
-      if (is_string($row['categories'])) $row['categories'] = explode(',', $row['categories']);
-      $categories = array_map(function($el) use ($mapping) {
-        return array_key_exists($el, $mapping) ? $this->mappingTableIds[$mapping[$el]]['idAndParentsId'] : [];
-      }, $row['categories']);
-      $data[$key]['categories'] = array_unique($this->array_flatten($categories));
+      if (is_array($row['categories'])) {
+        if (is_string($row['categories'])) $row['categories'] = explode(',', $row['categories']);
+        $categories = array_map(function($el) use ($mapping) {
+          return array_key_exists($el, $mapping) && array_key_exists($mapping[$el], $this->mappingTableIds) ? $this->mappingTableIds[$mapping[$el]]['idAndParentsId'] : [];
+        }, $row['categories']);
+        $data[$key]['categories'] = array_unique($this->array_flatten($categories));
+      }
     }
     return $data;
   }

src/Biopen/GeoDirectoryBundle/Services/ElementImportService.php

@@ -89,31 +89,6 @@ class ElementImportService
     $data = json_decode($json, true);
     if ($data === null) return null;
 
-    // data can be stored inside a data attribute
-    if (array_key_exists('data', $data)) $data = $data['data'];
-
-    foreach ($data as $key => $row) {
-			if (array_key_exists('geo', $row))
-			{
-				$data[$key]['latitude']  = $row['geo']['latitude'];
-				$data[$key]['longitude'] = $row['geo']['longitude'];
-				unset($data[$key]['geo']);
-			}
-			if (array_key_exists('address', $row))
-			{
-				$address = $row['address'];
-
-				if (gettype($address) == "string") $data[$key]['streetAddress'] = $address;
-				else if ($address) {
-					if (array_key_exists('streetAddress', $address))   $data[$key]['streetAddress']   = $address['streetAddress'];
-					if (array_key_exists('addressLocality', $address)) $data[$key]['addressLocality'] = $address['addressLocality'];
-					if (array_key_exists('postalCode', $address))      $data[$key]['postalCode']      = $address['postalCode'];
-					if (array_key_exists('addressCountry', $address))  $data[$key]['addressCountry']  = $address['addressCountry'];
-				}
-				unset($data[$key]['address']);
-			}
-		}
-
     if ($onlyGetData) return $data;
 
     $elementImportedCount = $this->importData($data, $import);
@@ -125,7 +100,7 @@ class ElementImportService
   // create a mapping table for ontology and taxonomy
   public function collectData($import)
   {
-		$data = $import->getUrl() ?$this->importJson($import, true) : $this->importCsv($import, true);
+		$data = $import->getUrl() ? $this->importJson($import, true) : $this->importCsv($import, true);
   	$this->mappingService->transform($data, $import);
   }
 

web/test.json

 {
-        "licence": "https://opendatacommons.org/licenses/odbl/summary/",
-        "ontology":"gogofull",
+  "licence": "https://opendatacommons.org/licenses/odbl/summary/",
+  "ontology":"gogofull",
 	"data":[{
 		"id":"Rhm",
 		"newfields": "balam",