Commit 3448f8db authored by Maxime Reyrolle's avatar Maxime Reyrolle

Merge branch...

Merge branch '994-creer-un-connecteur-permettant-d-extraire-les-donnees-a-partir-d-un-xpath-et-a-partir-de-l' into 'master'

Resolve "Créer un connecteur permettant d'extraire les données à partir d'un XPATH et à partir de l'équivalent en JSON"

Closes #994

See merge request libriciel/pole-plate-formes/pastell/pastell!551
parents 3d79ad60 a31979f1
Pipeline #10805 failed with stages
# 3.1.0
## Correction
## Évolutions
- Ajout d'une étape studio de transformation (création de meta-données ou de fichiers supplémentaires)
- Ajout d'un connecteur de transformation générique, permettant d'utiliser du Twig, du xpath et du jsonpath pour créer de nouvelles métadonnées #994
......
# Développement
```bash
git clone https://gitlab.libriciel.fr/libriciel/pole-plate-formes/pastell/pastell.git
docker-compose up -d
```
# Stratégie GIT
- pull (ou clone) et chechout master
......
......@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "96c169a4045f4705d565ffeab07ab084",
"content-hash": "95ff070066f291df72b1d4b1c22b3fda",
"packages": [
{
"name": "clue/stream-filter",
......@@ -505,6 +505,47 @@
],
"time": "2019-10-28T03:44:26+00:00"
},
{
"name": "flow/jsonpath",
"version": "0.5.0",
"source": {
"type": "git",
"url": "https://github.com/FlowCommunications/JSONPath.git",
"reference": "b9738858c75d008c1211612b973e9510f8b7f8ea"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/FlowCommunications/JSONPath/zipball/b9738858c75d008c1211612b973e9510f8b7f8ea",
"reference": "b9738858c75d008c1211612b973e9510f8b7f8ea",
"shasum": ""
},
"require": {
"php": ">=5.4.0"
},
"require-dev": {
"peekmo/jsonpath": "dev-master",
"phpunit/phpunit": "^7.0"
},
"type": "library",
"autoload": {
"psr-0": {
"Flow\\JSONPath": "src/",
"Flow\\JSONPath\\Test": "tests/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Stephen Frank",
"email": "stephen@flowsa.com"
}
],
"description": "JSONPath implementation for parsing, searching and flattening arrays",
"time": "2019-07-15T17:23:22+00:00"
},
{
"name": "flowjs/flow-php-server",
"version": "v1.0.3",
......@@ -3443,6 +3484,64 @@
],
"time": "2020-01-04T14:08:26+00:00"
},
{
"name": "symfony/polyfill-ctype",
"version": "v1.14.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-ctype.git",
"reference": "fbdeaec0df06cf3d51c93de80c7eb76e271f5a38"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/fbdeaec0df06cf3d51c93de80c7eb76e271f5a38",
"reference": "fbdeaec0df06cf3d51c93de80c7eb76e271f5a38",
"shasum": ""
},
"require": {
"php": ">=5.3.3"
},
"suggest": {
"ext-ctype": "For best performance"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.14-dev"
}
},
"autoload": {
"psr-4": {
"Symfony\\Polyfill\\Ctype\\": ""
},
"files": [
"bootstrap.php"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Gert de Pagter",
"email": "BackEndTea@gmail.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Symfony polyfill for ctype functions",
"homepage": "https://symfony.com",
"keywords": [
"compatibility",
"ctype",
"polyfill",
"portable"
],
"time": "2020-01-13T11:15:53+00:00"
},
{
"name": "symfony/polyfill-mbstring",
"version": "v1.14.0",
......@@ -3773,6 +3872,90 @@
"web"
],
"time": "2019-11-28T12:59:49+00:00"
},
{
"name": "twig/twig",
"version": "v3.0.4",
"source": {
"type": "git",
"url": "https://github.com/twigphp/Twig.git",
"reference": "582bdbdc173027ebfba3c93dc750a40b8f9ebc02"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/twigphp/Twig/zipball/582bdbdc173027ebfba3c93dc750a40b8f9ebc02",
"reference": "582bdbdc173027ebfba3c93dc750a40b8f9ebc02",
"shasum": ""
},
"require": {
"php": ">=7.2.5",
"symfony/polyfill-ctype": "^1.8",
"symfony/polyfill-mbstring": "^1.3"
},
"require-dev": {
"psr/container": "^1.0",
"symfony/phpunit-bridge": "^4.4.9|^5.0.9"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "3.0-dev"
}
},
"autoload": {
"psr-4": {
"Twig\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"BSD-3-Clause"
],
"authors": [
{
"name": "Fabien Potencier",
"email": "fabien@symfony.com",
"homepage": "http://fabien.potencier.org",
"role": "Lead Developer"
},
{
"name": "Twig Team",
"role": "Contributors"
},
{
"name": "Armin Ronacher",
"email": "armin.ronacher@active-4.com",
"role": "Project Founder"
}
],
"description": "Twig, the flexible, fast, and secure template language for PHP",
"homepage": "https://twig.symfony.com",
"keywords": [
"templating"
],
"funding": [
{
"url": "https://certification.symfony.com/",
"type": "custom"
},
{
"url": "https://live.symfony.com/",
"type": "custom"
},
{
"url": "https://symfony.com/cloud/",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/twig/twig",
"type": "tidelift"
}
],
"time": "2020-07-05T13:18:14+00:00"
}
],
"packages-dev": [
......@@ -4161,47 +4344,6 @@
"abandoned": "php-webdriver/webdriver",
"time": "2019-06-13T08:02:18+00:00"
},
{
"name": "flow/jsonpath",
"version": "0.3.4",
"source": {
"type": "git",
"url": "https://github.com/FlowCommunications/JSONPath.git",
"reference": "00aa9c361e4d0a210dd95f3c917a1e0dde3a957f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/FlowCommunications/JSONPath/zipball/00aa9c361e4d0a210dd95f3c917a1e0dde3a957f",
"reference": "00aa9c361e4d0a210dd95f3c917a1e0dde3a957f",
"shasum": ""
},
"require": {
"php": ">=5.4.0"
},
"require-dev": {
"peekmo/jsonpath": "dev-master",
"phpunit/phpunit": "^4.0"
},
"type": "library",
"autoload": {
"psr-0": {
"Flow\\JSONPath": "src/",
"Flow\\JSONPath\\Test": "tests/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Stephen Frank",
"email": "stephen@flowsa.com"
}
],
"description": "JSONPath implementation for parsing, searching and flattening arrays",
"time": "2016-09-06T17:43:18+00:00"
},
{
"name": "mikey179/vfsstream",
"version": "v1.6.8",
......@@ -6393,64 +6535,6 @@
"homepage": "https://symfony.com",
"time": "2020-02-14T07:42:58+00:00"
},
{
"name": "symfony/polyfill-ctype",
"version": "v1.14.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-ctype.git",
"reference": "fbdeaec0df06cf3d51c93de80c7eb76e271f5a38"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/fbdeaec0df06cf3d51c93de80c7eb76e271f5a38",
"reference": "fbdeaec0df06cf3d51c93de80c7eb76e271f5a38",
"shasum": ""
},
"require": {
"php": ">=5.3.3"
},
"suggest": {
"ext-ctype": "For best performance"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.14-dev"
}
},
"autoload": {
"psr-4": {
"Symfony\\Polyfill\\Ctype\\": ""
},
"files": [
"bootstrap.php"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Gert de Pagter",
"email": "BackEndTea@gmail.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Symfony polyfill for ctype functions",
"homepage": "https://symfony.com",
"keywords": [
"compatibility",
"ctype",
"polyfill",
"portable"
],
"time": "2020-01-13T11:15:53+00:00"
},
{
"name": "symfony/process",
"version": "v4.4.5",
......@@ -6716,5 +6800,6 @@
"ext-zip": "^1.15",
"ext-zend-opcache": "^7.2"
},
"platform-dev": []
"platform-dev": [],
"plugin-api-version": "1.1.0"
}
<?php
abstract class TransformationConnecteur extends Connecteur
{
abstract public function transform(DonneesFormulaire $donneesFormulaire, array $utilisateur_info): void;
}
<?php
class TransformationTransform extends ConnecteurTypeActionExecutor
{
/**
* @return bool
* @throws NotFoundException
* @throws UnrecoverableException
*/
public function go()
{
$donneesFormulaire = $this->getDonneesFormulaire();
/** @var TransformationConnecteur $transformationConnecteur */
$transformationConnecteur = $this->getConnecteur("transformation");
$utilisateur_info = $this->objectInstancier
->getInstance(DocumentActionEntite::class)
->getCreatorOfDocument(
$this->id_e,
$this->id_d
);
$transformationConnecteur->transform($donneesFormulaire, $utilisateur_info);
$message = "Transformation terminée";
$this->addActionOK($message);
$this->notify($this->action, $this->type, $message);
$this->setLastMessage($message);
return true;
}
}
<?php
require_once __DIR__ . "/lib/TransformationGeneriqueDefinition.class.php";
require_once __DIR__ . "/lib/SimpleTwigRenderer.class.php";
class TransformationGenerique extends TransformationConnecteur
{
/**
* @var DonneesFormulaire
*/
private $connecteurConfig;
private $transformationGeneriqueDefinition;
private $simpleTwigRenderer;
public function __construct(
TransformationGeneriqueDefinition $transformationGeneriqueDefinition,
SimpleTwigRenderer $simpleTwigRenderer
) {
$this->transformationGeneriqueDefinition = $transformationGeneriqueDefinition;
$this->simpleTwigRenderer = $simpleTwigRenderer;
}
public function setConnecteurConfig(DonneesFormulaire $donneesFormulaire)
{
$this->connecteurConfig = $donneesFormulaire;
}
/**
* @param DonneesFormulaire $donneesFormulaire
* @param array $utilisateur_info
*/
public function transform(DonneesFormulaire $donneesFormulaire, array $utilisateur_info): void
{
$result = $this->getNewValue($donneesFormulaire);
foreach ($result as $id => $value) {
$donneesFormulaire->setData($id, $value);
}
}
public function testTransform(DonneesFormulaire $donneesFormulaire): string
{
$result = $this->getNewValue($donneesFormulaire);
return json_encode($result);
}
/**
* @return array
*/
private function getNewValue(DonneesFormulaire $donneesFormulaire): array
{
$transformation_data = $this->transformationGeneriqueDefinition->getData($this->connecteurConfig);
foreach ($transformation_data as $element_id => $expression) {
$transformation_data[$element_id] = $this->simpleTwigRenderer->render(
$expression,
$donneesFormulaire
);
}
return $transformation_data;
}
}
<?php
require_once __DIR__ . "/../lib/TransformationGeneriqueDefinition.class.php";
class TransformationGeneriqueFillData extends ChoiceActionExecutor
{
/**
* @return bool
* @throws Exception
*/
public function go()
{
$definition_array = $this->getRecuperateur()->get('definition');
$id_element_array = $this->getRecuperateur()->get('id_element');
$data = [];
foreach ($id_element_array as $i => $id_element) {
$id_element = trim($id_element);
if (! $id_element) {
continue;
}
$data[$id_element] = $definition_array[$i] ?? "";
}
$transformationGeneriqueDefinition = $this->objectInstancier->getInstance(
TransformationGeneriqueDefinition::class
);
$transformationGeneriqueDefinition->setTransformation(
$this->getConnecteurConfig($this->id_ce),
$data
);
if ($this->getRecuperateur()->get('add_button') === 'add') {
$this->redirect("Connecteur/externalData?id_ce={$this->id_ce}&field={$this->field}");
exit;
}
return true;
}
/**
* @return bool
*/
public function display()
{
$fluxEntiteSQL = $this->objectInstancier->getInstance(FluxEntiteSQL::class);
$all_used = $fluxEntiteSQL->getUsedByConnecteur($this->id_ce, null, $this->id_e);
$flux = "";
if (count($all_used) == 1) {
$flux = $all_used[0]['flux'];
}
$documentType = $this->getDocumentTypeFactory()->getFluxDocumentType($flux);
$this->{'fieldsList'} = ($documentType->getFormulaire()->getFieldsList());
$this->{'flux'} = $flux;
$transformationGeneriqueDefinition = $this->objectInstancier->getInstance(
TransformationGeneriqueDefinition::class
);
$transformation_data = $transformationGeneriqueDefinition->getData(
$this->getConnecteurConfig($this->id_ce)
);
$transformation_data[''] = "";
$this->{'transformation_data'} = $transformation_data;
$this->renderPage(
"Données de transformation",
__DIR__ . "/../template/TransformationGeneriqueFillData.php"
);
return true;
}
/**
* @return bool
*/
public function displayAPI()
{
return false;
}
}
<?php
class TransformationGeneriqueTestExtraction extends ActionExecutor
{
/**
* @return bool
* @throws UnrecoverableException
*/
public function go(): bool
{
/** @var TransformationGenerique $connecteur */
$connecteur = $this->getMyConnecteur();
$donneesFormulaire = $this->getDonneesFormulaireFactory()->getNonPersistingDonneesFormulaire();
$result = $connecteur->testTransform($donneesFormulaire);
$this->setLastMessage("Résultat de l'extraction : " . $result);
return true;
}
}
nom: Transformation générique
type: transformation
formulaire:
page0:
definition:
name: "Définition de l'extraction"
read-only: true
type: file
visionneuse: TransformationGeneriqueVisionneuse
fill-data:
name: "Définition de l'extraction"
edit-only: true
type: externalData
link_name: Décrire les transformations
choice-action: fill-data
action:
test_extraction:
name: Tester la transformation
action-class: TransformationGeneriqueTestExtraction
fill-data:
name: Configurer les transformations
rule:
role_id_e: no-role
action-class: TransformationGeneriqueFillData
\ No newline at end of file
<?php
use Flow\JSONPath\JSONPath;
use Twig\Environment;
use Twig\Error\LoaderError;
use Twig\Error\SyntaxError;
use Twig\Extension\SandboxExtension;
use Twig\Loader\ArrayLoader;
use Twig\Sandbox\SecurityPolicy;
use Twig\TwigFunction;
class SimpleTwigRenderer
{
public const XPATH_FUNCTION = 'xpath';
public const JSONPATH_FUNCION = 'jsonpath';
private const AUTHORIZED_TWIG_TAGS = ['if','for'];
private const AUHTORIZED_TWIG_FILTERS = ['escape'];
private const AUHTORIZED_TWIG_METHODS = [];
private const AUHTORIZED_TWIG_PROPERTIES = [];
private const AUHTORIZED_TWIG_FUNCTIONS = [];
/**