diff --git a/Framework/advanced-aggregator.json b/Framework/advanced-aggregator.json index c0d6fe83e0..196af8c106 100644 --- a/Framework/advanced-aggregator.json +++ b/Framework/advanced-aggregator.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" @@ -19,7 +19,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" }, "infologger": { "": "Configuration of the Infologger (optional).", "filterDiscardDebug": "true", "": "Set to true to discard debug and trace messages (default: false)", diff --git a/Framework/advanced-external-histo.json b/Framework/advanced-external-histo.json index ec93326a4f..9d6e2e5f2e 100644 --- a/Framework/advanced-external-histo.json +++ b/Framework/advanced-external-histo.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" @@ -19,7 +19,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" } }, "tasks": { diff --git a/Framework/advanced.json b/Framework/advanced.json index 5a37ec8702..287e46078b 100644 --- a/Framework/advanced.json +++ b/Framework/advanced.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" @@ -19,7 +19,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" }, "infologger": { "": "Configuration of the Infologger (optional).", "filterDiscardDebug": "true", "": "Set to true to discard debug and trace messages (default: false)", diff --git a/Framework/basic-aggregator.json b/Framework/basic-aggregator.json index a5c553741f..393a23ca38 100644 --- a/Framework/basic-aggregator.json +++ b/Framework/basic-aggregator.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" @@ -19,7 +19,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" }, "infologger": { "": "Configuration of the Infologger (optional).", "filterDiscardDebug": "false", "": "Set to 1 to discard debug and trace messages (default: false)", diff --git a/Framework/basic-external-histo.json b/Framework/basic-external-histo.json index bbf7e78d99..000e727dee 100644 --- a/Framework/basic-external-histo.json +++ b/Framework/basic-external-histo.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" @@ -19,7 +19,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" } }, "tasks": { diff --git a/Framework/basic-functional.json.in b/Framework/basic-functional.json.in index 2d47226ca6..1005d656c4 100644 --- a/Framework/basic-functional.json.in +++ b/Framework/basic-functional.json.in @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" @@ -23,7 +23,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" } }, "tasks": { diff --git a/Framework/basic-no-sampling.json b/Framework/basic-no-sampling.json index 221ce7205c..1b99cf4b90 100644 --- a/Framework/basic-no-sampling.json +++ b/Framework/basic-no-sampling.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" @@ -19,7 +19,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" } }, "tasks": { diff --git a/Framework/basic.json b/Framework/basic.json index 1d8ff36da3..9f6a0cd252 100644 --- a/Framework/basic.json +++ b/Framework/basic.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable", @@ -16,14 +16,11 @@ "passName": "", "": "Pass type - e.g. spass, cpass1", "provenance": "qc", "": "Provenance - qc or qc_mc depending whether it is normal data or monte carlo data" }, - "monitoring": { - "url": "infologger:///debug?qc" - }, "consul": { "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" }, "infologger": { "": "Configuration of the Infologger (optional).", "filterDiscardDebug": "false", "": "Set to true to discard debug and trace messages (default: false)", diff --git a/Framework/batch-test.json.in b/Framework/batch-test.json.in index b0541e6e15..1390193372 100644 --- a/Framework/batch-test.json.in +++ b/Framework/batch-test.json.in @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" @@ -23,7 +23,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" } }, "tasks": { diff --git a/Framework/example-default.json b/Framework/example-default.json index 07bd335def..3d44d058fa 100644 --- a/Framework/example-default.json +++ b/Framework/example-default.json @@ -19,7 +19,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" } }, "tasks": { diff --git a/Framework/include/QualityControl/CommonSpec.h b/Framework/include/QualityControl/CommonSpec.h index 3beb320820..6691127baf 100644 --- a/Framework/include/QualityControl/CommonSpec.h +++ b/Framework/include/QualityControl/CommonSpec.h @@ -41,7 +41,7 @@ struct CommonSpec { int activityOriginalNumber = 0; std::string monitoringUrl = "infologger:///debug?qc"; std::string consulUrl; - std::string conditionDBUrl = "http://ccdb-test.cern.ch:8080"; + std::string conditionDBUrl = "http://ali-qcdb-test.cern.ch:8080"; LogDiscardParameters infologgerDiscardParameters; double postprocessingPeriod = 30.0; std::string bookkeepingUrl; diff --git a/Framework/multiNode.json b/Framework/multiNode.json index 59c53f87ec..d8a7a09587 100644 --- a/Framework/multiNode.json +++ b/Framework/multiNode.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" @@ -19,7 +19,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" } }, "tasks": { diff --git a/Framework/multinode-test.json.in b/Framework/multinode-test.json.in index 2e71d4d04f..afe15a9728 100644 --- a/Framework/multinode-test.json.in +++ b/Framework/multinode-test.json.in @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" @@ -21,7 +21,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" } }, "tasks": { diff --git a/Framework/postprocessing-async.json b/Framework/postprocessing-async.json index c7fdb9cecc..b1b566fc70 100644 --- a/Framework/postprocessing-async.json +++ b/Framework/postprocessing-async.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" @@ -22,7 +22,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" }, "postprocessing": { "periodSeconds": "0.0001" diff --git a/Framework/postprocessing.json b/Framework/postprocessing.json index e72cd83e18..e01ba800b7 100644 --- a/Framework/postprocessing.json +++ b/Framework/postprocessing.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" @@ -19,7 +19,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" }, "postprocessing": { "periodSeconds": "10" diff --git a/Framework/readout-no-sampling.json b/Framework/readout-no-sampling.json index 82612f9a80..76a88d4637 100644 --- a/Framework/readout-no-sampling.json +++ b/Framework/readout-no-sampling.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" @@ -19,7 +19,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" } }, "tasks": { diff --git a/Framework/readout.json b/Framework/readout.json index c3f68b70f5..6a09bfde64 100644 --- a/Framework/readout.json +++ b/Framework/readout.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" @@ -19,7 +19,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" } }, "tasks": { diff --git a/Framework/src/runRepositoryBenchmark.cxx b/Framework/src/runRepositoryBenchmark.cxx index 821784360e..3b8b084e14 100644 --- a/Framework/src/runRepositoryBenchmark.cxx +++ b/Framework/src/runRepositoryBenchmark.cxx @@ -29,8 +29,8 @@ void addCustomOptions(bpo::options_description& options) "Maximum number of iterations of Run/ConditionalRun/OnData (0 - infinite, default : 3)")( "number-tasks", bpo::value()->default_value(0), "Informative only, the number of tasks being ran in parallel.")( - "database-url", bpo::value()->default_value("ccdb-test.cern.ch:8080"), - "Database url (default : ccdb-test.cern.ch:8080")("database-username", bpo::value()->default_value(""), + "database-url", bpo::value()->default_value("ali-qcdb-test.cern.ch:8083"), + "Database url (default : ali-qcdb-test.cern.ch:8083")("database-username", bpo::value()->default_value(""), "Database username (default : )")( "database-password", bpo::value()->default_value(""), "Database password (default : )")( "database-name", bpo::value()->default_value(""), "Database name (default : ")( diff --git a/Framework/test/testCheckWorkflow.json b/Framework/test/testCheckWorkflow.json index 6c9640bb93..0803ebf6c4 100644 --- a/Framework/test/testCheckWorkflow.json +++ b/Framework/test/testCheckWorkflow.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" diff --git a/Framework/test/testSharedConfig.json b/Framework/test/testSharedConfig.json index 818227cc96..2d0fc60e03 100644 --- a/Framework/test/testSharedConfig.json +++ b/Framework/test/testSharedConfig.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" @@ -16,7 +16,7 @@ "url": "infologger:///debug?qc" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "localhost:8083" } }, "tasks": { diff --git a/Framework/test/testTaskInterface.cxx b/Framework/test/testTaskInterface.cxx index c44351aedb..e955ecf29c 100644 --- a/Framework/test/testTaskInterface.cxx +++ b/Framework/test/testTaskInterface.cxx @@ -201,14 +201,14 @@ TEST_CASE("retrieveCondition") bad.addBadChannel(3, o2::emcal::BadChannelMap::MaskType_t::DEAD_CELL); std::map meta; o2::ccdb::CcdbApi api; - api.init("ccdb-test.cern.ch:8080"); + api.init("ali-qcdb-test.cern.ch:8083"); api.storeAsTFileAny(&bad, "qc/TST/conditions", meta); // retrieve it TaskRunnerConfig taskConfig; auto* objectsManager = new ObjectsManager(taskConfig.name, taskConfig.className, taskConfig.detectorName, 0); test::TestTask testTask(objectsManager); - testTask.setCcdbUrl("ccdb-test.cern.ch:8080"); + testTask.setCcdbUrl("ali-qcdb-test.cern.ch:8083"); o2::emcal::BadChannelMap* bcm = testTask.testRetrieveCondition(); CHECK(bcm->getChannelStatus(1) == o2::emcal::BadChannelMap::MaskType_t::GOOD_CELL); CHECK(bcm->getChannelStatus(3) == o2::emcal::BadChannelMap::MaskType_t::DEAD_CELL); diff --git a/Framework/test/testThrowNameClash.json b/Framework/test/testThrowNameClash.json index 71c638ccba..a73030f229 100644 --- a/Framework/test/testThrowNameClash.json +++ b/Framework/test/testThrowNameClash.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" diff --git a/Framework/test/testWorkflow.json b/Framework/test/testWorkflow.json index f02db53386..a5c0cb2be5 100644 --- a/Framework/test/testWorkflow.json +++ b/Framework/test/testWorkflow.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" diff --git a/Modules/Daq/etc/daq.json b/Modules/Daq/etc/daq.json index b85850e62d..7687a3fcc3 100644 --- a/Modules/Daq/etc/daq.json +++ b/Modules/Daq/etc/daq.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" diff --git a/Modules/Example/etc/every-object.json b/Modules/Example/etc/every-object.json index be6536ceda..25931247bc 100644 --- a/Modules/Example/etc/every-object.json +++ b/Modules/Example/etc/every-object.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "localhost:8083", "username": "not_applicable", "password": "not_applicable", "name": "not_applicable" diff --git a/doc/QCDB.md b/doc/QCDB.md index e355e8180a..34fff67650 100644 --- a/doc/QCDB.md +++ b/doc/QCDB.md @@ -185,6 +185,7 @@ The address of the CCDB will have to be updated in the Tasks config file. At the moment, the description of the REST api can be found in this document : +And the full documentation for a local CCDB setup is here: https://docs.google.com/document/d/1_GM6yY7ejVEIRi1y8Ooc9ongrGgZyCiks6Ca0OAEav8/edit?tab=t.0 --- diff --git a/doc/QuickStart.md b/doc/QuickStart.md index 56d47802e7..f588c0df95 100644 --- a/doc/QuickStart.md +++ b/doc/QuickStart.md @@ -22,7 +22,7 @@ Thanks! ## Requirements -A RHEL9 machine or a Mac (although you might have some issues running Readout). Ubuntu is only supported on a best-effort basis. +A RHEL9 or RHEL10 machine or a Mac (although you might have some issues running Readout). Ubuntu is only supported on a best-effort basis. ## Setup @@ -34,11 +34,10 @@ A RHEL9 machine or a Mac (although you might have some issues running Readout). 2. Prepare the QualityControl development package * `aliBuild init QualityControl@master --defaults o2` -4. Build/install the QualityControl. +3. Build/install the QualityControl. * `aliBuild build QualityControl --defaults o2` * At this point you might encounter a message about missing system requirements. Run `aliDoctor --defaults o2 QualityControl` to get a full information about what is missing and how to install it. - -Note: you can also use the alibuild "defaults" called `o2-dataflow` to avoid building simulation related packages. If you plan to use `Readout`, you can build both with one `aliBuild` command by listing them one after another, space-separated. + * Note: you can also use the alibuild "defaults" called `o2-dataflow` to avoid building simulation related packages. If you plan to use `Readout`, you can build both with one `aliBuild` command by listing them one after another, space-separated. ### Environment loading @@ -47,12 +46,29 @@ Whenever you want to work with O2 and QualityControl, do To load multiple independent packages, list them one after the other, space-separated (e.g. `alienv enter QualityControl/latest Readout/latest`). O2 is automatically pulled by QualityControl, thus no need to specify it explicitly. +### Repository + +In this guide we use a repository centrally hosted at CERN and called `ali-qcdb-test`. + +To access it from CERN, one can simply use the URL `ali-qcdb-test.cern.ch:8083`. However, many people develop and test their QC from outside, and therefore, we recommend the following approach. + +1. Create a tunnel (**always keep it running while working on your QC**) + + ssh -L 8083:ali-qcdb-test.cern.ch:8083 @lxplus.cern.ch +2. Check that it worked by accessing `http://localhost:8083/browse` in your browser. +3. Use the URL `localhost:8083` to access the repository, whether in your config files or in the browser to access its web interface. + +*It is a test and development instance. Data is regularly cleaned up and there is no guarantee on the data!* + +#### Alternatives + +1. the `ccdb-test`, if you have a working grid certificate (`ccdb-test.cern.ch:8080`), +2. a local instance ([easy installation instructions](QCDB.md#local-ccdb-setup)). + ## Execution To make sure that your system is correctly setup, we are going to run a basic QC workflow attached to a simple data producer. -The QC repository and the GUI are central services, thus they don't have to be installed on your machine. If you want to set them up on your computer or in your lab, please have a look at the [Local CCDB setup](QCDB.md#local-ccdb-setup) section. - ### Basic workflow This is the workflow we want to run: @@ -66,7 +82,7 @@ This is the workflow we want to run: - Finally the _Checker_ is also in charge of storing the resulting _MonitorObject_ into the repository where it will be accessible by the web GUI. - It also pushes it to a _Printer_ for the sake of this tutorial. -To run it simply do: +Then in another terminal, simply run: o2-qc-run-basic @@ -83,16 +99,22 @@ This is not the GUI we will use to see the histograms. The example above consists of a single DPL workflow containing both the main processing and the QC infrastructure. In a more real case, we would prefer attaching the QC without modifying the original topology. It can be done by merging two (or more) workflows with a pipe (`|`), as shown below: o2-qc-run-producer | o2-qc --config json://${QUALITYCONTROL_ROOT}/etc/basic.json - + ![basic-schema-2-exe](images/basic-schema-2-exe.png) This command uses two executables. The first one contains only the _Producer_ (see Figure above), which represents the data flow to which we want to apply the QC. The second executable generates the QC workflow based on the given configuration file (more details in a few sections). These two workflows are joined together using the pipe `|` character. This example illustrates how to add QC to any DPL workflow by using `o2-qc` and passing it a configuration file. -#### Repository and GUI +#### QC web GUI + +We have already seen how to access the `ali-qcdb-test` from your computer. However, there is a better tool to see your plots: the QC web GUI (QCG). + +From CERN, simply go to [https://ali-qcg-test.cern.ch/?page=objectTree](https://ali-qcg-test.cern.ch/?page=objectTree). + +From outside CERN, follow [these instructions](https://alice-flp.docs.cern.ch/Developers/KB/dynamic_forwarding/) before accessing [https://ali-qcg-test.cern.ch/?page=objectTree](https://ali-qcg-test.cern.ch/?page=objectTree). -The plots are stored in the [ccdb-test](ccdb-test.cern.ch:8080/browse) at CERN. If everything works fine you should see the objects being published in the QC web GUI (QCG) at this address: [https://qcg-test.cern.ch/?page=objectTree](https://qcg-test.cern.ch/?page=objectTree). The link brings you to the hierarchy of objects (see screenshot below). Open "qc/TST/MO/QcTask" (the task you are running) and click on "example" which is the name of your histogram. +The link brings you to the hierarchy of objects (see screenshot below). Open "qc/TST/MO/QcTask" (the task you are running) and click on "example" which is the name of your histogram. If you click on the item in the tree again, you will see an updated version of the plot. @@ -131,7 +153,7 @@ In another terminal window run the ExampleTrend post-processing task, as follows o2-qc-run-postprocessing --config json://${QUALITYCONTROL_ROOT}/etc/postprocessing.json --id ExampleTrend ``` -On the [QCG website](https://qcg-test.cern.ch/?page=objectTree) you will see a TTree and additional plots visible under the path `/qc/TST/MO/ExampleTrend`. They show how different properties of the Example histogram change during time. The longer the applications are running, the more data will be visible. +On the [QCG website](#qc-web-gui) you will see a TTree and additional plots visible under the path `/qc/TST/MO/ExampleTrend`. They show how different properties of the Example histogram change during time. The longer the applications are running, the more data will be visible. The [post-processing component](doc/PostProcessing.md) and its convenience classes allow to trend and correlate various characteristics of histograms and other data structures generated by QC tasks and checks. One can create their own post-processing tasks or use the ones included in the framework and configure them for one's own needs. diff --git a/doc/benchmark-repo.md b/doc/benchmark-repo.md index 368b700c70..5d74f47770 100644 --- a/doc/benchmark-repo.md +++ b/doc/benchmark-repo.md @@ -25,7 +25,7 @@ o2-qc-repo-benchmark --max-iterations 30 --database-name "" --database-username "" --database-password "" - --database-url ccdb-test.cern.ch:8080 + --database-url ali-qcdb-test.cern.ch:8083 --monitoring-threaded 0 --monitoring-threaded-interval 5 ```