From fdbc72bb052194e04b8339fb4fa316fcbe706321 Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Tue, 14 Nov 2023 23:01:15 -0100 Subject: [PATCH] +files metadata Signed-off-by: Maxence Lange --- .../app_upgrade_guide/upgrade_to_28.rst | 26 +- developer_manual/client_apis/WebDAV/basic.rst | 4 - .../digging_deeper/files-metadata.rst | 383 ++++++++++++++++++ developer_manual/digging_deeper/index.rst | 1 + 4 files changed, 409 insertions(+), 5 deletions(-) create mode 100644 developer_manual/digging_deeper/files-metadata.rst diff --git a/developer_manual/app_publishing_maintenance/app_upgrade_guide/upgrade_to_28.rst b/developer_manual/app_publishing_maintenance/app_upgrade_guide/upgrade_to_28.rst index eeecd3c2c..949659dc6 100644 --- a/developer_manual/app_publishing_maintenance/app_upgrade_guide/upgrade_to_28.rst +++ b/developer_manual/app_publishing_maintenance/app_upgrade_guide/upgrade_to_28.rst @@ -141,10 +141,28 @@ Added APIs * ``\OCP\IPhoneNumberUtil::convertToStandardFormat`` to convert input into an E164 formatted phone number. See :ref:`phonenumberutil` for an example. * ``\OCP\IPhoneNumberUtil::getCountryCodeForRegion`` to get the E164 country code for a given region. See :ref:`phonenumberutil` for an example. * ``\OCP\AppFramework\Http\EmptyContentSecurityPolicy::allowEvalWasm(bool)``: sets ``wasm-unsafe-eval`` in ``script-src`` of the Content Security Policy `to allow compilation and execution of WebAssembly on the page `_ +* ``\OCP\FilesMetadata\IMetadataBackgroundEvent::getNode`` +* ``\OCP\FilesMetadata\IMetadataBackgroundEvent::getMetadata`` +* ``\OCP\FilesMetadata\IMetadataLiveEvent::getNode`` +* ``\OCP\FilesMetadata\IMetadataLiveEvent::getMetadata`` +* ``\OCP\FilesMetadata\IMetadataLiveEvent::requestBackgroundJob`` +* ``\OCP\FilesMetadata\IFilesMetadataManager::refreshMetadata`` +* ``\OCP\FilesMetadata\IFilesMetadataManager::getMetadata`` +* ``\OCP\FilesMetadata\IFilesMetadataManager::saveMetadata`` +* ``\OCP\FilesMetadata\IFilesMetadataManager::deleteMetadata`` +* ``\OCP\FilesMetadata\IFilesMetadataManager::getMetadataQuery`` +* ``\OCP\FilesMetadata\IFilesMetadataManager::getKnownMetadata`` +* ``\OCP\FilesMetadata\IFilesMetadataManager::initMetadata`` +* ``\OCP\FilesMetadata\IMetadataQuery::retrieveMetadata`` +* ``\OCP\FilesMetadata\IMetadataQuery::extractMetadata`` +* ``\OCP\FilesMetadata\IMetadataQuery::joinIndex`` +* ``\OCP\FilesMetadata\IMetadataQuery::getMetadataKeyField`` +* ``\OCP\FilesMetadata\IMetadataQuery::getMetadataValueField`` * ``wasm-unsafe-eval`` is `supported by most browsers `_ * WebAssembly compilation and execution in worker threads is not affected by this directive (browsers allow compilation and execution of WebAssembly in worker threads by default) + Changed APIs ^^^^^^^^^^^^ @@ -202,6 +220,8 @@ Added events * Typed event ``OCP\DB\Events\AddMissingPrimaryKeyEvent`` to add missing indices to the database schema. * Typed event ``OCP\Files\Events\NodeAddedToFavorite`` was added * Typed event ``OCP\Files\Events\NodeRemovedFromFavorite`` was added +* Typed event ``OCP\FilesMetadata\Event\MetadataBackgroundEvent`` was added +* Typed event ``OCP\FilesMetadata\Event\MetadataLiveEvent`` was added * Typed event ``OCP\Share\Events\BeforeShareCreatedEvent`` was added * Typed event ``OCP\Share\Events\BeforeShareDeletedEvent`` was added * Typed event ``OCP\Share\Events\ShareAcceptedEvent`` was added @@ -280,7 +300,11 @@ Removed events - +Removed WebDAV properties +^^^^^^^^^^^^^^^^^^^^^^^^^ + +* +* diff --git a/developer_manual/client_apis/WebDAV/basic.rst b/developer_manual/client_apis/WebDAV/basic.rst index a54122a76..17c9376e7 100644 --- a/developer_manual/client_apis/WebDAV/basic.rst +++ b/developer_manual/client_apis/WebDAV/basic.rst @@ -307,10 +307,6 @@ Supported properties +-------------------------------+-------------------------------------------------+--------------------------------------------------------------------------------------+ | | The token of the lock. | ``files_lock/0e53dfb6-61b4-46f0-b38e-d9a428292998`` | +-------------------------------+-------------------------------------------------+--------------------------------------------------------------------------------------+ -| | The pixel-size of an image. | ``{"width":451,"height":529}`` | -+-------------------------------+-------------------------------------------------+--------------------------------------------------------------------------------------+ -| | The GPS location of a file taken from EXIF data.| ``{"latitude":52.52093727702034,"longitude":13.397258069518614}``| -+-------------------------------+-------------------------------------------------+--------------------------------------------------------------------------------------+ Listing folders (rfc4918_) -------------------------- diff --git a/developer_manual/digging_deeper/files-metadata.rst b/developer_manual/digging_deeper/files-metadata.rst new file mode 100644 index 000000000..0b378665c --- /dev/null +++ b/developer_manual/digging_deeper/files-metadata.rst @@ -0,0 +1,383 @@ +============== +Files Metadata +============== + +.. versionadded:: 28 + +Nextcloud includes an API to manage files' metadata, with a deep integration in WebDAV. + + +Concept overview +---------------- + +When a file is created or modified on Nextcloud, a refresh of its metadata is initiated. You will then be able to listen to the `MetadataLiveEvent` event to create, update, or delete metadata. If you suspect your process to be hungry in time or resources, you can request for a background event called `MetadataBackgroundEvent` and do your work there. + +Consuming the Files Metadata API +-------------------------------- + +To consume the Files Metadata API, you will need to :ref:`inject` ``IFilesMetadataManager``. +This manager offers the following methods: + + * ``refreshMetadata(Node $node, int $process)`` This method will initiate the process of refreshing the metadata related to a file. + * ``getMetadata(int $fileId, bool $generate)`` This method will returns a ``IFilesMetadata`` which contains metadata related to a file. + * ``saveMetadata(IFilesMetadata $filesMetadata)`` Save a ``IFilesMetadata`` and generate indexes. + * ``deleteMetadata(int $fileId)`` Delete all metadata related to a file. + * ``getMetadataQuery(IQueryBuilder $qb, string $fileTableAlias, string $fileIdField)`` This method will returns a ``IMetadataQuery`` to help building Sql request. + * ``getKnownMetadata()`` Returns a list of known metadata keys globally available to the instance, and the expected type for each value. + * ``initMetadata(string $key, string $type, bool $indexed, bool $remotelyEditable)`` This method allow to initiate a metadata before any of the files is examined + + +Live & Background Events +------------------------ + +Two (2) events can be caught by your app to generate metadata. The first one is called on the main process, just after the upload/modification of the file. +The second one is called on a background process, initiated by cronjob. + + * ``OCP\FilesMetadata\Event\MetadataLiveEvent`` + * ``OCP\FilesMetadata\Event\MetadataBackgroundEvent`` + +Both events contain those methods: + + * ``getNode()`` Returns related ``Node``. + * ``getMetadata()`` Returns ``IFilesMetadata``. Any changes made to this object will be saved at the end of the event. + +Generating metadata from a file might require a lot of resources; in that case it is advised to run this generation +on a background job by requesting a re-run by calling the method ``MetadataLiveEvent::requestBackgroundJob()`` + +.. code-block:: php + :emphasize-lines: 23, 24 + + registerEventListener(MetadataLiveEvent::class, UpdateFilesMetadata::class); + $context->registerEventListener(MetadataBackgroundEvent::class, UpdateFilesMetadata::class); + } + + public function boot(IBootContext $context): void { + } + } + +.. note:: + If the generation of metadata requires low resources, the app only need to listen on ``MetadataLiveEvent`` + +.. code-block:: php + + getNode(); + + // my-first-meta is light enough + $metadata = $event->getMetadata(); + $metadata->setString('my-first-meta', 'yes'); + + if ($event instanceof MetadataLiveEvent) { + $event->requestBackgroundJob(); + return; + } + + // my-second-meta is too heavy and should be run on a background job + $metadata->setInt('my-second-meta', 1234, true); + } + } + + +Read metadata using occ command +------------------------------- + +Stored metadata related to a file can be obtained from a console, using the ``occ`` command: + +.. code-block:: console + + $ ./occ metadata:get 1742 + { + "my-first-meta": { + "value": "yes", + "type": "string", + "indexed": false + }, + "my-second-meta": { + "value": 1234, + "type": "int", + "indexed": true + } + } + +.. note:: + The generation process can also be initiated, using the ``--refresh`` option. Please note that the file owner need to be specified in the command. + When refreshing metadata from the console, both ``MetadataLiveEvent`` and ``MetadataBackgroundEvent`` are triggered, without waiting for the next tick of crontab + + +Update metadata using PROPPATCH +------------------------------- + +Using WebDAV request, a client can create or update metadata about a file: + +.. code-block:: console + + curl 'https://cloud.example.net/remote.php/dav/files/test/document.txt' \ + --user test:test \ + --request PROPPATCH \ + --data ' + + + + 123 + + + ' + +This will return a result like + +.. code-block:: xml + + + + + /remote.php/dav/files/test/document.txt + + + + + HTTP/1.1 200 OK + + + + +.. note:: + WebDAV prefixes metadata with ``getServerContainer()->get(IFilesMetadataManager::class); + $metadataManager->initMetadata('myapp-test', IMetadataValueWrapper::TYPE_INT, true, IMetadataValueWrapper::EDIT_REQ_OWNERSHIP); + } + + +Retrieve metadata using PROPFIND +-------------------------------- + +Metadata are available to the WebDAV PROPFIND requests: + +.. code-block:: console + + curl 'https://cloud.example.net/remote.php/dav/files/test/document.txt' \ + --user test:test \ + --request PROPFIND \ + --data ' + + + + + ' + + +This will return a result like + +.. code-block:: xml + + + + + /remote.php/dav/files/test/document.txt + + + 123 + + HTTP/1.1 200 OK + + + + + + +WebDAV SEARCH based on metadata +------------------------------- + +.. code-block:: console + :emphasize-lines: 8-10, 19-32, 36-39 + + curl 'https://cloud.example.net/remote.php/dav/' \ + --user test:test \ + --request SEARCH \ + --data ' + + + + + + + + + + /files/test/ + infinity + + + + + + + + + 10 + + + + + + 1000 + + + + + + + + + + + + + 200 + 0 + + + ' + + +This will return a result like: + +.. code-block:: xml + + + + + /remote.php/dav/files/test/ + + + + + HTTP/1.1 404 Not Found + + + + /remote.php/dav/files/test/document.txt + + + 123 + + HTTP/1.1 200 OK + + + + /remote.php/dav/files/test/another-one.txt + + + 369 + + HTTP/1.1 200 OK + + + + + +.. warning:: + Metadata used in ORDER and WHERE statement will require metadata to be initiated and set as indexed. + It is required to call ``IFilesMetadataManager::initMetadata()`` before running the WebDAV request or it will returns an exception because of unknown property. + +.. code-block:: php + + /** lib/AppInfo/Application.php */ + public function boot(IBootContext $context): void { + /** @var IFilesMetadataManager $metadataManager */ + $metadataManager = $context->getServerContainer()->get(IFilesMetadataManager::class); + $metadataManager->initMetadata('my-second-meta', IMetadataValueWrapper::TYPE_INT, true); + } + + +Metadata Query Helper +--------------------- + +``IFilesMetadataManager::getMetadataQuery(IQueryBuilder $qb, string $fileTableAlias, string $fileIdField)`` returns a ``IMetadataQuery`` to help building Sql request with the following methods. +Parameters when calling the method are the alias of the table, and the name of the field, that contains file ids. + + * ``retrieveMetadata()`` will add a select on the stored metadata + * ``extractMetadata(array $row)`` convert a fetched row from the resultinto a ``IFilesMetadata`` + * ``joinIndex(string $metadataKey, bool $enforce)`` join the indexes to the request + * ``getMetadataKeyField(string $metadataKey)`` returns the field and the aliased table of the key of an index + * ``getMetadataValueField(string $metadataKey)`` returns the field and the aliased table of the value of an index + + +.. code-block:: php + + // generate your normal query builder + $qb = new QueryBuilder(); + $qb->select('file_id') + ->from('my_table', 'my_alias'); + + /** @var IFilesMetadataManager $metadataManager */ + $metadataManager = $context->getServerContainer()->get(IFilesMetadataManager::class); + + // get a configured query helper and add a select on the metadata + $metadataQuery = $metadataManager->getMetadataQuery($qb, 'my_alias', 'file_id'); + $metadataQuery->retrieveMetadata(); + + // right join the index table and get only value lower than 8910 for metadata 'my-second-meta' + $metadataQuery->joinIndex('my-second-meta', true); + $qb->where($qb->expr()->lt($metadataQuery->getMetadataValueField('my-second-meta'), $qb->createNamedParameter(8910))); + + // get result + $result = $qb->execute(); + $items = $result->fetchAll(); + + // extract metadata from each row + $entries = array_map(function (array $data) use ($metadataQuery): array { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); + }, $items); + diff --git a/developer_manual/digging_deeper/index.rst b/developer_manual/digging_deeper/index.rst index c610dbad2..d0145cdcc 100644 --- a/developer_manual/digging_deeper/index.rst +++ b/developer_manual/digging_deeper/index.rst @@ -11,6 +11,7 @@ Digging deeper classloader continuous_integration email + files-metadata flow http_client javascript-apis