ElementImportMappingService.php 7.95 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
<?php

namespace Biopen\GeoDirectoryBundle\Services;

use Doctrine\ODM\MongoDB\DocumentManager;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Helper\ProgressBar;
use Biopen\GeoDirectoryBundle\Document\Element;
use Biopen\GeoDirectoryBundle\Document\ElementStatus;
use Biopen\GeoDirectoryBundle\Document\ModerationState;
use Biopen\GeoDirectoryBundle\Document\Coordinates;
use Biopen\GeoDirectoryBundle\Document\Option;
use Biopen\GeoDirectoryBundle\Document\OptionValue;
use Biopen\GeoDirectoryBundle\Document\UserInteractionContribution;
use Biopen\GeoDirectoryBundle\Document\PostalAddress;
use Biopen\GeoDirectoryBundle\Document\ElementUrl;
use Biopen\GeoDirectoryBundle\Document\ElementImage;
use Biopen\GeoDirectoryBundle\Document\ImportState;
use Biopen\CoreBundle\Document\GoGoLogImport;
use Biopen\CoreBundle\Document\GoGoLogLevel;

class ElementImportMappingService
{
  protected $import;
25 26
  protected $createMissingOptions;
  protected $parentCategoryIdToCreateMissingOptions;
27
  protected $em;
28
  protected $coreFields = ['id', 'name', 'categories', 'streetAddress', 'addressLocality', 'postalCode', 'addressCountry', 'latitude', 'longitude', 'images', 'owner', 'source'];
Sebastian Castro's avatar
Sebastian Castro committed
29 30 31 32 33 34 35 36 37 38
  protected $mappedCoreFields = [
    'title' => 'name',
    'taxonomy' => 'categories',
    'address' => 'streetAddress',
    'city' => 'addressLocatily',
    'postcode' => 'postalCode',
    'country' => 'addressCountry',
    'lat' => 'latitude',
    'long' => 'longitude', 'lng' => 'longitude'
  ];
39

40
  public function __construct(DocumentManager $documentManager)
41
  {
42
    $this->em = $documentManager;
43 44 45 46 47
  }

  public function transform($data, $import)
  {
    $this->import = $import;
48 49 50 51
    $this->createMissingOptions = $import->getCreateMissingOptions();
    $parent = $import->getParentCategoryToCreateOptions() ?: $this->em->getRepository('BiopenGeoDirectoryBundle:Category')->findOneByIsRootCategory(true);
    $this->parentCategoryIdToCreateMissingOptions = $parent->getId();

52
    $this->collectOntology($data, $import);
53
    $data = $this->mapOntology($data);
Sebastian Castro's avatar
Sebastian Castro committed
54

55 56
    // remove empty row, i.e. without name
    $data = array_filter($data, function($row) { return array_key_exists('name', $row); });
Sebastian Castro's avatar
Sebastian Castro committed
57 58 59 60 61 62 63
    // $data = $this->addMissingFieldsToData($data);

    if ($import->isCategoriesFieldMapped())
    {
      $this->collectTaxonomy($data, $import);
      $data = $this->mapTaxonomy($data);
    }
64

65 66
    $this->em->persist($import);
    $this->em->flush();
67 68 69
    return $data;
  }

70 71 72
  public function collectOntology($data, $import)
  {
    $ontologyMapping = $import->getOntologyMapping();
73
    $allNewFields = [];
74 75 76
    foreach($data as $row)
    {
      foreach ($row as $key => $value) {
77
        if (!in_array($key, $allNewFields)) $allNewFields[] = $key;
78 79
        if (!array_key_exists($key, $ontologyMapping)) {
          $value = in_array($key, $this->coreFields) ? $key : "";
Sebastian Castro's avatar
Sebastian Castro committed
80
          if (!$value && in_array($this->mappedCoreFields[$key], $this->coreFields)) $value = $this->mappedCoreFields[$key];
81 82 83 84
          $ontologyMapping[$key] = $value;
        }
      }
    }
85 86 87 88 89
    // delete no more used fields
    foreach($ontologyMapping as $field => $mappedField) {
      if (!in_array($field, $allNewFields)) unset($ontologyMapping[$field]);
    }

90 91 92 93 94 95
    $import->setOntologyMapping($ontologyMapping);
  }

  public function collectTaxonomy($data, $import)
  {
    $taxonomyMapping = $import->getTaxonomyMapping();
96
    $allNewCategories = [];
97 98 99 100
    $this->createOptionsMappingTable();

    foreach($data as $row)
    {
101 102 103 104
      $categories = $row['categories'];
      $categories = is_array($categories) ? $categories : explode(',', $categories);
      foreach($categories as $category) {
        if (!in_array($category, $allNewCategories)) $allNewCategories[] = $category;
105 106
        if ($category && !array_key_exists($category, $taxonomyMapping))
        {
107 108
          $categorySlug = $this->slugify($category);
          $value = array_key_exists($categorySlug, $this->mappingTableIds) ? $this->mappingTableIds[$categorySlug]['id'] : '';
109 110 111 112

          // create option if does not exist
          if ($value == '' && $this->createMissingOptions) $value = $this->createOption($category);

113 114
          $taxonomyMapping[$category] = $value;
        }
115 116 117 118 119 120
        // create options for previously imported non mapped options
        if (array_key_exists($category, $taxonomyMapping)
            && (!$taxonomyMapping[$category] || $taxonomyMapping[$category] == '/')
            && $this->createMissingOptions) {
          $taxonomyMapping[$category] = $this->createOption($category);
        }
121 122
      }
    }
123 124 125 126
    // delete no more used categories
    foreach($taxonomyMapping as $category => $mappedCategory) {
      if (!in_array($category, $allNewCategories)) unset($taxonomyMapping[$category]);
    }
127 128 129
    $import->setTaxonomyMapping($taxonomyMapping);
  }

130 131 132 133 134 135
  private function mapOntology($data)
  {
    $mapping = $this->import->getOntologyMapping();

    foreach ($data as $key => $row) {
      foreach ($mapping as $search => $replace) {
136 137 138 139
        if ($replace == '/' || $replace == '') {
          unset($data[$key][$search]);
        }
        else if (isset($row[$search]) && !isset($row[$replace])) {
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158
          $data[$key][$replace] = $data[$key][$search];
          unset($data[$key][$search]);
        }
      }
    }
    return $data;
  }

  private function addMissingFieldsToData($data)
  {
    foreach ($data as $key => $row) {
      $missingFields = array_diff($this->coreFields, array_keys($row));
      foreach ($missingFields as $missingField) {
        $data[$key][$missingField] = "";
      }
    }
    return $data;
  }

159
  private function mapTaxonomy($data)
160
  {
161
    $mapping = $this->import->getTaxonomyMapping();
162 163 164
    foreach ($data as $key => $row)
    {
      if (is_string($row['categories'])) $row['categories'] = explode(',', $row['categories']);
165 166 167
      $data[$key]['categories'] = array_map(function($el) use ($mapping) {
        return array_key_exists($el, $mapping) ? $mapping[$el] : '';
      }, $row['categories']);
168
    }
169
    return $data;
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
  }

  private function createOptionsMappingTable($options = null)
  {
    if ($options === null) $options = $this->em->getRepository('BiopenGeoDirectoryBundle:Option')->findAll();

    foreach($options as $option)
    {
      $ids = [
        'id' => $option->getId(),
        'idAndParentsId' => $option->getIdAndParentOptionIds()
      ];
      $this->mappingTableIds[$this->slugify($option->getNameWithParent())] = $ids;
      $this->mappingTableIds[$this->slugify($option->getName())] = $ids;
      $this->mappingTableIds[strval($option->getId())] = $ids;
      if ($option->getCustomId()) $this->mappingTableIds[$this->slugify($option->getCustomId())] = $ids;
    }
  }

  private function createOption($name)
  {
    $option = new Option();
    $option->setName($name);
    $parent = $this->em->getRepository('BiopenGeoDirectoryBundle:Category')->find($this->parentCategoryIdToCreateMissingOptions);
    $option->setParent($parent);
    $option->setUseIconForMarker(false);
    $option->setUseColorForMarker(false);
    $this->em->persist($option);
    $this->createOptionsMappingTable([$option]);
199
    return $option->getId();
200 201 202 203
  }

  private function slugify($text)
  {
204
    if (!is_string($text)) return;
205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
    // replace non letter or digits by -
    $text = str_replace('é', 'e', $text);
    $text = str_replace('è', 'e', $text);
    $text = str_replace('ê', 'e', $text);
    $text = str_replace('ô', 'o', $text);
    $text = str_replace('ç', 'c', $text);
    $text = str_replace('à', 'a', $text);
    $text = str_replace('â', 'a', $text);
    $text = str_replace('î', 'i', $text);
    $text = preg_replace('~[^\pL\d]+~u', '-', $text);

    $text = iconv('utf-8', 'us-ascii//TRANSLIT', $text); // transliterate
    $text = preg_replace('~[^-\w]+~', '', $text); // remove unwanted characters
    $text = trim($text, '-'); // trim
    $text = rtrim($text, 's'); // remove final "s" for plural
    $text = preg_replace('~-+~', '-', $text); // remove duplicate -
    $text = strtolower($text); // lowercase

    if (empty($text)) return '';
    return $text;
  }
}