{"type":"doc","content":[{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"At this point, the following steps should already be achieved by the Data Integration (DI) team:","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Create the JSONL files ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://boxalino.atlassian.net/wiki/spaces/BPKB/pages/252149803/Data+Integration#1.-Export-JSONL-files"}},{"text":" ","type":"text"}]}]}]}]},{"type":"paragraph","content":[{"text":"The ","type":"text"},{"text":"load request","type":"text","marks":[{"type":"em"}]},{"text":" is done","type":"text"},{"text":" for every document JSONL (following the required ","type":"text","marks":[{"type":"strong"}]},{"text":"Data Structure","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"https://boxalino.atlassian.net/wiki/spaces/BPKB/pages/252280881"}}]},{"text":")","type":"text","marks":[{"type":"strong"}]},{"text":" required for your Data Integration process, once they are generated.","type":"text"}]},{"type":"panel","attrs":{"panelType":"note"},"content":[{"type":"paragraph","content":[{"text":"For a product data sync, make sure the following tables have been prepared and are provided: ","type":"text"},{"text":"doc_product","type":"text","marks":[{"type":"link","attrs":{"href":"/wiki/spaces/BPKB/pages/252149870"}}]},{"text":", ","type":"text"},{"text":"doc_attributes","type":"text","marks":[{"type":"link","attrs":{"href":"https://boxalino.atlassian.net/wiki/spaces/BPKB/pages/252280945"}}]},{"text":", ","type":"text"},{"text":"doc_attribute_values","type":"text","marks":[{"type":"link","attrs":{"href":"https://boxalino.atlassian.net/wiki/spaces/BPKB/pages/252313624"}}]},{"text":" and ","type":"text"},{"text":"doc_languages","type":"text","marks":[{"type":"link","attrs":{"href":"https://boxalino.atlassian.net/wiki/spaces/BPKB/pages/252280975"}}]}]}]},{"type":"paragraph"},{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"toc","parameters":{"macroParams":{},"macroMetadata":{"macroId":{"value":"740ae37e-292e-4a4a-8bbd-5e20984b3b00"},"schemaVersion":{"value":"1"},"title":"Table of Contents"}},"localId":"573943cb-a088-4da3-a0f2-52a9ac0b1e3d"}},{"type":"heading","attrs":{"level":2},"content":[{"text":" /load ","type":"text"}]},{"type":"paragraph","content":[{"text":"The ","type":"text"},{"text":"load","type":"text","marks":[{"type":"strong"}]},{"text":" request will do the following things:","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":" It saves your JSONL (for every data type required for the data sync type) in a Google Cloud Storage (GCS) Bucket from the ","type":"text"},{"text":"Boxalino project","type":"text","marks":[{"type":"textColor","attrs":{"color":"#36b37e"}},{"type":"strong"}]}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"the files are available for 30 days in the clients` GCS bucket","type":"text"}]}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"It loads every JSONL in a BigQuery table in ","type":"text"},{"text":"Boxalino project","type":"text","marks":[{"type":"textColor","attrs":{"color":"#36b37e"}},{"type":"strong"}]}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"for ","type":"text"},{"text":"Instant","type":"text","marks":[{"type":"em"}]},{"text":" Updates : the tables have a retention policy of ","type":"text"},{"text":"1 day","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"for ","type":"text"},{"text":"Delta","type":"text","marks":[{"type":"em"}]},{"text":" Updates: the tables have a retention policy of ","type":"text"},{"text":"7 days","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"for ","type":"text"},{"text":"Full","type":"text","marks":[{"type":"em"}]},{"text":" Updates: the tables have a retention policy of ","type":"text"},{"text":"15 days","type":"text","marks":[{"type":"strong"}]}]}]}]}]}]},{"type":"panel","attrs":{"panelType":"note"},"content":[{"type":"paragraph","content":[{"text":"Boxalino will provide access to the data Storage Buckets and BigQuery datasets that store your data.","type":"text"}]}]},{"type":"paragraph","content":[{"text":"The service response is a JSON payload as documented here ","type":"text","marks":[{"type":"strong"}]},{"type":"inlineCard","attrs":{"url":"https://boxalino.atlassian.net/wiki/spaces/BPKB/pages/747208705"}},{"text":" ","type":"text","marks":[{"type":"strong"}]}]},{"type":"panel","attrs":{"panelType":"error"},"content":[{"type":"paragraph","content":[{"text":"If the service response is an error like: ","type":"text"},{"text":"413 Request Entity Too Large","type":"text","marks":[{"type":"code"}]},{"text":" - please use the ","type":"text"},{"text":"/load/chunk","type":"text","marks":[{"type":"code"}]},{"text":" flow.","type":"text"}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Request Definition","type":"text"}]},{"type":"table","attrs":{"layout":"default","isNumberColumnEnabled":true,"width":760.0,"localId":"9b18fecb-bd24-4000-ad8b-dc8d839637a8"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph","content":[{"text":"Endpoint","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"full data sync","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"https://boxalino-di-full-krceabfwya-ew.a.run.app ","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"delta data sync","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"https://boxalino-di-delta-krceabfwya-ew.a.run.app","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"instant-update data sync","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"https://boxalino-di-instant-update-krceabfwya-ew.a.run.app ","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"stage / testing","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"https://boxalino-di-stage-krceabfwya-ew.a.run.app ","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph","content":[{"text":"Action","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableCell","attrs":{"colspan":2,"rowspan":1,"colwidth":[145.0,463.0]},"content":[{"type":"paragraph","content":[{"text":"/load","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph","content":[{"text":"Method","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableCell","attrs":{"colspan":2,"rowspan":1,"colwidth":[145.0,463.0]},"content":[{"type":"paragraph","content":[{"text":"POST","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph","content":[{"text":"Body","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableCell","attrs":{"colspan":2,"rowspan":1,"colwidth":[145.0,463.0]},"content":[{"type":"paragraph","content":[{"text":"the document JSONL","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph","content":[{"text":"Headers","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"Authorization","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"Basic ","type":"text"},{"text":"base64","type":"text","marks":[{"type":"em"}]},{"text":"<>","type":"text"},{"type":"hardBreak"},{"type":"hardBreak"},{"text":"Note: only the API credentials with role=ADMIN are valid. ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://boxalino.atlassian.net/wiki/spaces/BPKB/pages/713785345/API+Credentials#Roles"}},{"text":" ","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"Content-Type","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"application/json","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"doc","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"the document type (as described in ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://boxalino.atlassian.net/wiki/spaces/BPKB/pages/252280881"}},{"text":" )","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"client","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"your Boxalino account name","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"dev","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"only add it if the ","type":"text"},{"text":"dev ","type":"text","marks":[{"type":"em"}]},{"text":"data index must be updated","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"mode","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"D","type":"text","marks":[{"type":"strong"}]},{"text":" for delta , ","type":"text"},{"text":"I ","type":"text","marks":[{"type":"strong"}]},{"text":"for instant update, ","type":"text"},{"text":"F","type":"text","marks":[{"type":"strong"}]},{"text":" for full","type":"text"}]},{"type":"paragraph","content":[{"text":"technical: ","type":"text","marks":[{"type":"em"}]},{"text":"the constants from the GcpClientInterface ","type":"text"},{"text":"https://github.com/boxalino/data-integration-doc-php/blob/3.0.0/src/Service/GcpRequestInterface.php#L18","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/boxalino/data-integration-doc-php/blob/3.0.0/src/Service/GcpRequestInterface.php#L18"}}]},{"text":" ","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"type","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"integration type (product, order, etc)","type":"text"},{"type":"hardBreak"},{"text":"https://github.com/boxalino/data-integration-doc-php/blob/3.0.0/src/Service/GcpRequestInterface.php#L22","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/boxalino/data-integration-doc-php/blob/3.0.0/src/Service/GcpRequestInterface.php#L22"}}]},{"text":" ","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"tm","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"time, in format: ","type":"text"},{"text":"YmdHis","type":"text","marks":[{"type":"strong"}]},{"type":"hardBreak"},{"text":"requirement: ","type":"text","marks":[{"type":"strong"}]},{"text":"the same ","type":"text"},{"text":"tm","type":"text","marks":[{"type":"strong"}]},{"text":" value must be used from the begging of the DI process until the end, for all files.","type":"text"}]},{"type":"paragraph","content":[{"text":"technical","type":"text","marks":[{"type":"em"}]},{"text":": used to identify the version of the documents and create the content. ","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"ts","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"(optional) timestamp, must be ","type":"text"},{"text":"millisecond","type":"text","marks":[{"type":"strong"}]},{"text":" based in ","type":"text"},{"text":"UNIX timestamp","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"technical","type":"text","marks":[{"type":"em"}]},{"text":": calculated from the time the data has been exported to BQ; the update requests will be applied in version ts order; ","type":"text"},{"text":"https://github.com/boxalino/data-integration-doc-php/blob/3.0.0/src/Service/Flow/DiRequestTrait.php#L140","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/boxalino/data-integration-doc-php/blob/3.0.0/src/Service/Flow/DiRequestTrait.php#L140"}}]},{"text":" ","type":"text"},{"type":"hardBreak"},{"text":"if it is not set, our system sets it","type":"text","marks":[{"type":"em"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"chunk","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"(optional) for loading content by chunks","type":"text"},{"type":"hardBreak"},{"text":"see ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://boxalino.atlassian.net/wiki/spaces/BPKB/pages/415432770/Load+Request#Load-By-Chunks"}},{"text":" ","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"dataset","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"(optional) the dataset in which the doc_X tables must be stored;","type":"text"}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"if not provided, the service will check the ","type":"text"},{"text":"_ ","type":"text","marks":[{"type":"strong"}]},{"text":"dataset in the Boxalino project, to which you will have access","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"project","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"(optional) the project name where the documents are to be stored;","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[109.0]},"content":[{"type":"paragraph"}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[145.0]},"content":[{"type":"paragraph","content":[{"text":"bucket","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[463.0]},"content":[{"type":"paragraph","content":[{"text":"(optional) the storage bucket where the doc_X will be loaded;","type":"text"}]},{"type":"paragraph","content":[{"text":"if not provided, the service will use the Boxalino project.","type":"text"}]}]}]}]},{"type":"paragraph","content":[{"text":"A","type":"text"},{"text":" LOAD REQUEST ","type":"text","marks":[{"type":"strong"}]},{"text":"code-sample is available in the ","type":"text"},{"text":"data-integration-doc-php","type":"text","marks":[{"type":"em"}]},{"text":" library: ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://github.com/boxalino/data-integration-doc-php/blob/3.0.0/src/Service/Flow/LoadTrait.php"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":"Optionally, if you are to use an ","type":"text"},{"text":"advanced rest client","type":"text","marks":[{"type":"em"},{"type":"link","attrs":{"href":"https://chrome.google.com/webstore/detail/advanced-rest-client/hgmloofddffdnphfgcellkdfbfbjeloo"}}]},{"text":" , you can configure it as such:","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":1930,"alt":"Advanced REST Client SYNC request structure","id":"97c145a8-309c-48bf-9d44-ca6743f8ea6f","collection":"contentId-415432770","type":"file","height":1132}}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Integration Strategy","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"doc_attribute","type":"text","marks":[{"type":"em"}]},{"text":", ","type":"text"},{"text":"doc_attribute_value","type":"text","marks":[{"type":"em"}]},{"text":", ","type":"text"},{"text":"doc_language","type":"text","marks":[{"type":"em"}]},{"text":" can be synchronized with a call to the ","type":"text"},{"text":"load","type":"text","marks":[{"type":"strong"}]},{"text":" endpoint","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"because GCP has a size limit of 32MB for POST requests, do not use the ","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ff5630"}},{"type":"strong"}]},{"text":"/load","type":"text","marks":[{"type":"code"}]},{"text":" ","type":"text"},{"text":"service for big data exports","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ff5630"}},{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"if the data is too big, the service returns a ","type":"text"},{"text":"413 ENTITY TOO LARGE","type":"text","marks":[{"type":"strong"}]},{"text":" response. In this case, switch to the batch load ","type":"text"}]}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"doc_product","type":"text","marks":[{"type":"em"}]},{"text":" / ","type":"text"},{"text":"doc_order / doc_user / doc_content / etc and other content bigger than 32 MB ","type":"text","marks":[{"type":"em"}]},{"text":"must be loaded in batches (max size of 32MB) or via GCS Public Signed URL ","type":"text"},{"text":"/load/chunk","type":"text","marks":[{"type":"code"}]},{"text":" endpoint","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"NOTE:","type":"text","marks":[{"type":"strong"}]},{"text":" when using ","type":"text"},{"text":"/load/chunk","type":"text","marks":[{"type":"code"}]},{"text":" service to generate a Google Cloud Storage Public URL, ","type":"text"},{"text":"there is no size limit!","type":"text","marks":[{"type":"strong"}]},{"text":" It is possible to load all data at once using one GCS Public URL link.","type":"text"}]}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Load in Batches / data > 32 MB ","type":"text"}]},{"type":"paragraph","content":[{"text":"For content over 32MB, we provide an endpoint to access a ","type":"text"},{"text":"Signed GCS Url","type":"text","marks":[{"type":"strong"}]},{"text":" that would put all your streamed content into a file (currently there is no defined file size limit in GCS)","type":"text"}]},{"type":"panel","attrs":{"panelType":"note"},"content":[{"type":"paragraph","content":[{"text":"Read more about Google Cloud Signed URL ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://cloud.google.com/storage/docs/access-control/signed-urls"}},{"text":" (response samples, uses, etc)","type":"text"}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"1. Make a request for public upload link","type":"text"}]},{"type":"paragraph","content":[{"text":"This is the generic POST request:","type":"text"}]},{"type":"codeBlock","content":[{"text":"curl --connect-timeout 60 --max-time 300 \"https://boxalino-di--krceabfwya-ew.a.run.app/load/chunk\" \\\n -X POST \\\n -H \"Content-Type: application/json\" \\\n -H \"client: \" \\\n -H \"dev: true|false\" \\\n -H \"tm: YYYYmmddHHiiss\" \\\n -H \"type: product|content|order|user|communication_history|communication_planning|user_generated_content\" \\\n -H \"mode: \" \\\n -H \"chunk: \" \\\n -H \"doc: doc_product|doc_order|doc_user|doc_content|..\" \\\n -H \"Authorization: Basic \"","type":"text"}]},{"type":"paragraph","content":[{"text":"The response will be an upload link that can only be used in order to create the file in the clients` GCS bucket. ","type":"text"},{"text":"The link is valid for 30 minutes.","type":"text","marks":[{"type":"strong"}]}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"NOTE: ","type":"text","marks":[{"type":"strong"}]},{"text":"The HTTP headers ","type":"text"},{"text":"must","type":"text","marks":[{"type":"strong"}]},{"text":" include a new property - ","type":"text"},{"text":"chunk","type":"text","marks":[{"type":"strong"}]},{"text":". The ","type":"text"},{"text":"chunk ","type":"text","marks":[{"type":"strong"}]},{"text":"value (number or textual) is the order of the batch / pagination / SEEK strategy used for content segmentation / etc. It is part of the final file available in GCS, to ensure that the Signed GCS Url is unique.","type":"text"}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"2. Upload the content on the public link","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"A code sample is available ","type":"text"},{"text":"in our generic PHP library","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/boxalino/data-integration-doc-php/blob/4.0.0/src/Service/Flow/LoadByChunkTrait.php"}}]}]},{"type":"codeBlock","content":[{"text":"curl --connect-timeout 60 --timeout 0 \\\n -X PUT \\\n -H \"Content-Type: application/octet-stream\" \\\n -d \"\"","type":"text"}]},{"type":"panel","attrs":{"panelIconId":"1f44c","panelIcon":":ok_hand:","panelIconText":"👌","panelColor":"#FFEBE6","panelType":"custom"},"content":[{"type":"paragraph","content":[{"text":"The use of the header ","type":"text"},{"text":"chunk","type":"text","marks":[{"type":"code"}]},{"text":" is required if the same file/document is exported in batches/sections. ","type":"text"},{"type":"hardBreak"},{"text":"Repeat steps 1+2 for every data batch loaded in GCS. ","type":"text"},{"type":"hardBreak"},{"text":"Make sure to increment the value of the ","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ff5630"}},{"type":"strong"}]},{"text":"chunk","type":"text","marks":[{"type":"code"}]},{"text":" ","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ff5630"}}]},{"text":"property for each ","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ff5630"}},{"type":"strong"}]},{"text":"/load/chunk","type":"text","marks":[{"type":"code"}]},{"text":" request.","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ff5630"}},{"type":"strong"}]}]}]},{"type":"panel","attrs":{"panelType":"success"},"content":[{"type":"paragraph","content":[{"text":"Step #1 - #2 must be repeated for every batch that is required to be added for the given process (same tm, mode & type)","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Only after all the data was loaded, you can move on to step#3.","type":"text","marks":[{"type":"textColor","attrs":{"color":"#006644"}},{"type":"strong"}]}]}]},{"type":"paragraph"},{"type":"heading","attrs":{"level":4},"content":[{"text":"3. Load data to BigQuery","type":"text"}]},{"type":"paragraph","content":[{"text":"Make an HTTP POST call to ","type":"text"},{"text":"load/bq endpoint","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"https://github.com/boxalino/data-integration-doc-php/blob/3.0.0/src/Service/GcpRequestInterface.php#L14"}}]},{"text":". ","type":"text","marks":[{"type":"strong"}]},{"text":"A code sample is available ","type":"text"},{"text":"in our generic PHP library","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/boxalino/data-integration-doc-php/blob/4.0.0/src/Service/Flow/LoadBqTrait.php"}}]}]},{"type":"paragraph","content":[{"text":"This is the generic POST request:","type":"text"}]},{"type":"codeBlock","content":[{"text":"curl --connect-timeout 60 --max-time 300 \"https://boxalino-di--krceabfwya-ew.a.run.app/load/bq\" \\\n -X POST \\\n -H \"Content-Type: application/json\" \\\n -H \"client: \" \\\n -H \"dev: true|false\" \\\n -H \"tm: YYYYmmddHHiiss\" \\\n -H \"type: product|content|order|user|communication_history|communication_planning|user_generated_content\" \\\n -H \"mode: \" \\\n -H \"doc: doc_product|doc_order|doc_user|doc_content|..\" \\\n -H \"Authorization: Basic \"","type":"text"}]},{"type":"paragraph"},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"The HTTP headers must have the required properties as described in the ","type":"text"},{"text":"Request Definition","type":"text","marks":[{"type":"link","attrs":{"href":"https://boxalino.atlassian.net/wiki/spaces/BPKB/pages/415432770/Load+Request#Request-Definition"}}]}]},{"type":"paragraph","content":[{"text":"data-integration-doc-php/DiRequestTrait.php at 3.2.0 · boxalino/data-integration-doc-php","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/boxalino/data-integration-doc-php/blob/3.2.0/src/Service/Flow/DiRequestTrait.php#L198"}}]},{"text":" ","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Fallback / retry policies","type":"text"}]},{"type":"paragraph","content":[{"text":"We do our best to ensure that our services are continuously available to receive requests. The endpoints have a timeout of 3600s. This means that your POST request will receive a response when the service finished your request (either error or succes) or when the timeout is reached.","type":"text"}]},{"type":"paragraph","content":[{"text":"The ","type":"text"},{"text":"/load","type":"text","marks":[{"type":"code"}]},{"text":" , ","type":"text"},{"text":"/load/chunk","type":"text","marks":[{"type":"code"}]},{"text":" , ","type":"text"},{"text":"/load/bq","type":"text","marks":[{"type":"code"}]},{"text":" have a succesfull response on average in 10sec (so the wait time is minimal). Howether, we recommend for your fallback/retry policy, to ","type":"text"},{"text":"use a ","type":"text","marks":[{"type":"textColor","attrs":{"color":"#006644"}},{"type":"strong"}]},{"text":"timeout","type":"text","marks":[{"type":"code"}]},{"text":" configuration of 3-5 min","type":"text","marks":[{"type":"textColor","attrs":{"color":"#006644"}},{"type":"strong"}]},{"text":".","type":"text"},{"type":"hardBreak"},{"type":"hardBreak"},{"text":"Based on your system, the ","type":"text"},{"text":"timeout","type":"text","marks":[{"type":"code"}]},{"text":" is represented by different parameters for the HTTP request.","type":"text"}]},{"type":"codeBlock","content":[{"text":"curl --max-time 10 --connect-timeout 30 -X POST -d ","type":"text"}]},{"type":"paragraph","content":[{"text":"--max-time","type":"text","marks":[{"type":"code"}]},{"text":" is the timeout: wait 10s for a response from the server or","type":"text"},{"text":" stop connection","type":"text","marks":[{"type":"em"}]},{"type":"hardBreak"},{"text":"--connect-timeout","type":"text","marks":[{"type":"code"}]},{"text":" is the connection timeout: wait 30s to make a connection to the service or ","type":"text"},{"text":"stop connection","type":"text","marks":[{"type":"em"}]},{"type":"hardBreak"},{"type":"hardBreak"},{"text":"Instead of ","type":"text"},{"text":"stop connection","type":"text","marks":[{"type":"em"}]},{"text":", we recommend to integrate a fallback policy with a retry.","type":"text"},{"type":"hardBreak"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Using private GCP resources for DI","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"The following requirements must be followed if you want to ","type":"text"},{"text":"save the JSONL (#1) ","type":"text","marks":[{"type":"em"}]},{"text":"and ","type":"text"},{"text":"load the JSONL (#2)","type":"text","marks":[{"type":"em"}]},{"text":" in ","type":"text"},{"text":"your custom project","type":"text","marks":[{"type":"strong"}]},{"text":". ","type":"text"}]},{"type":"paragraph","content":[{"text":"If you are happy to use the Boxalino service for content storage & processing, you can ignore the requirements list.","type":"text"}]}]},{"type":"paragraph","content":[{"text":"For the case when the client wants to use their own GCP resources for the Data Integration with Boxalino, the following requirements must be complied with:","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"BigQuery Datasets","type":"text"}]},{"type":"panel","attrs":{"panelType":"note"},"content":[{"type":"paragraph","content":[{"text":"Using different datasets for different data integration processes (delta, full, instant) will allow the set of ","type":"text"},{"text":"default table expiration.","type":"text","marks":[{"type":"em"}]},{"text":" By doing so, you are following BigQuery Storage Optimization best practices as described in the ","type":"text"},{"text":"documentation","type":"text","marks":[{"type":"link","attrs":{"href":"https://cloud.google.com/bigquery/docs/best-practices-storage"}}]},{"text":".","type":"text"}]}]},{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}]},{"type":"heading","attrs":{"level":4},"marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"1. Naming","type":"text"}]},{"type":"paragraph","content":[{"text":"The BigQuery dataset in which your account documents are loaded can be named after the ","type":"text"},{"text":"data index and process ","type":"text","marks":[{"type":"strong"}]},{"text":"it is meant to synchronize to","type":"text"},{"text":":","type":"text","marks":[{"type":"strong"}]}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"if the ","type":"text"},{"text":"request","type":"text","marks":[{"type":"em"}]},{"text":" is for the ","type":"text"},{"text":"dev","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ff5630"}},{"type":"em"}]},{"text":" data index - ","type":"text"},{"text":"","type":"text","marks":[{"type":"textColor","attrs":{"color":"#4c9aff"}},{"type":"strong"}]},{"text":"_dev_","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ff5630"}},{"type":"strong"}]},{"text":"","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ffc400"}},{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"if the ","type":"text"},{"text":"request","type":"text","marks":[{"type":"em"}]},{"text":" is for the production data index - ","type":"text"},{"text":"","type":"text","marks":[{"type":"textColor","attrs":{"color":"#4c9aff"}},{"type":"strong"}]},{"text":"_","type":"text","marks":[{"type":"strong"}]},{"text":"","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ffc400"}},{"type":"strong"}]}]}]}]},{"type":"paragraph","content":[{"text":", where:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":" ","type":"text","marks":[{"type":"strong"}]},{"text":"","type":"text","marks":[{"type":"textColor","attrs":{"color":"#4c9aff"}},{"type":"strong"}]},{"text":" is the account name","type":"text","marks":[{"type":"strong"}]},{"text":" provided by Boxalino.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ffc400"}},{"type":"strong"}]},{"text":" is the process F","type":"text","marks":[{"type":"strong"}]},{"text":" (for full)","type":"text"},{"text":", D ","type":"text","marks":[{"type":"strong"}]},{"text":"(for delta)","type":"text"},{"text":", I ","type":"text","marks":[{"type":"strong"}]},{"text":"(for instant)","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Example, for our an account ","type":"text"},{"text":"boxalino_client","type":"text","marks":[{"type":"textColor","attrs":{"color":"#4c9aff"}},{"type":"strong"}]},{"text":", the following datasets must exist (upon your integration use-cases): ","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"for ","type":"text"},{"text":"dev","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ff5630"}}]},{"text":" index: ","type":"text"},{"text":"boxalino_client","type":"text","marks":[{"type":"textColor","attrs":{"color":"#4c9aff"}}]},{"text":"_","type":"text"},{"text":"dev","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ff5630"}},{"type":"strong"}]},{"text":"_","type":"text"},{"text":"I","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ffc400"}}]},{"text":", ","type":"text"},{"text":"boxalino_client","type":"text","marks":[{"type":"textColor","attrs":{"color":"#4c9aff"}}]},{"text":"_","type":"text"},{"text":"dev","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ff5630"}}]},{"text":"_","type":"text"},{"text":"F","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ffc400"}}]},{"text":", ","type":"text"},{"text":"boxalino_client","type":"text","marks":[{"type":"textColor","attrs":{"color":"#4c9aff"}}]},{"text":"_","type":"text"},{"text":"dev","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ff5630"}}]},{"text":"_","type":"text"},{"text":"D ","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ffc400"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"for production data index: ","type":"text"},{"text":"boxalino_client","type":"text","marks":[{"type":"textColor","attrs":{"color":"#4c9aff"}}]},{"text":"_","type":"text"},{"text":"I","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ffc400"}}]},{"text":", ","type":"text"},{"text":"boxalino_client","type":"text","marks":[{"type":"textColor","attrs":{"color":"#4c9aff"}}]},{"text":"_","type":"text"},{"text":"F","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ffc400"}}]},{"text":", ","type":"text"},{"text":"boxalino_client","type":"text","marks":[{"type":"textColor","attrs":{"color":"#4c9aff"}}]},{"text":"_","type":"text"},{"text":"D ","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ffc400"}}]}]}]}]},{"type":"paragraph","content":[{"text":"The above datasets must exist in your project.","type":"text"}]},{"type":"paragraph"},{"type":"heading","attrs":{"level":4},"marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"2. Location","type":"text"}]},{"type":"paragraph","content":[{"text":"Upon the creation of your dataset, please use the ","type":"text"},{"text":"Data Location: EU","type":"text","marks":[{"type":"strong"}]},{"type":"hardBreak"}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":512,"id":"302473f2-161e-4677-aeac-9fc249f42d71","collection":"contentId-415432770","type":"file","height":50}}]},{"type":"heading","attrs":{"level":4},"marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"3. Permissions","type":"text"}]},{"type":"paragraph","content":[{"text":"In order to have ","type":"text"},{"text":"read & write","type":"text","marks":[{"type":"em"}]},{"text":" access to your private dataset, please provide the following permissions to the Data Integration Service Account (DISA) ","type":"text"},{"text":"55483703770-compute@developer.gserviceaccount.com","type":"text","marks":[{"type":"code"}]},{"text":":","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"BigQuery Data Viewer","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"BigQuery Metadata Viewer","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"BigQuery Data Editor / Owner","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"BigQuery Job User","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"or BigQuery Admin to the created datasets ","type":"text"},{"text":"datasets.","type":"text","marks":[{"type":"em"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Storage Bucket ","type":"text"}]},{"type":"heading","attrs":{"level":4},"marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"1. Naming","type":"text"}]},{"type":"paragraph","content":[{"text":"In your custom project must be available the storage buckets in which #1 step can be done (loading your generated JSONL document).","type":"text"}]},{"type":"paragraph","content":[{"text":"Follow the ","type":"text"},{"text":" Google documentation on how to create storage buckets.","type":"text","marks":[{"type":"link","attrs":{"href":"https://cloud.google.com/storage/docs/creating-buckets"}}]}]},{"type":"paragraph","content":[{"text":"The storage buckets have the requirement to be unique within the scope of the ","type":"text"},{"text":"integration. ","type":"text","marks":[{"type":"em"}]},{"text":" Please use the following naming formula: ","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"__dev ","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"_","type":"text","marks":[{"type":"strong"}]},{"text":" ","type":"text"}]}]}]},{"type":"paragraph"},{"type":"heading","attrs":{"level":4},"marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"2. Location","type":"text"}]},{"type":"paragraph","content":[{"text":"For content storage, please use either ","type":"text"},{"text":"Multi-region (eu)","type":"text","marks":[{"type":"em"}]},{"text":" or ","type":"text"},{"text":"Region (europe-west1).","type":"text","marks":[{"type":"em"}]}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":426,"id":"39b05273-7718-4e17-9172-06b6233383d5","collection":"contentId-415432770","type":"file","height":81}}]},{"type":"paragraph"},{"type":"heading","attrs":{"level":4},"marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"3. Permissions","type":"text"}]},{"type":"paragraph","content":[{"text":"In order to have ","type":"text"},{"text":"load & read","type":"text","marks":[{"type":"em"}]},{"text":" access to your private Google Cloud Storage buckets, please provide the following permissions to the Data Integration Service Account (DISA) ","type":"text"},{"text":"55483703770-compute@developer.gserviceaccount.com","type":"text","marks":[{"type":"code"}]},{"text":":","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Storage Object Creator","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Storage Object Admin ","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"The above permissions can be replaced with the ","type":"text"},{"text":"Storage Admin","type":"text","marks":[{"type":"strong"}]},{"text":" role.","type":"text"}]},{"type":"paragraph","content":[{"type":"hardBreak"}]}],"version":1}

Browser not supported