Node Steering

Steer a task to a particular server or group of servers.

  5 minute read  

Steering Tags

While no action is required to perform tasks in the cluster, it is possible to steer requests to a specific server or a group of servers. This is accomplished by providing an array of server tags specified in the server.tags field for the server in the application.properties file in combination with nodeTags provided on the request itself.

This example will assume we have two servers. The first server is configured on the server side as a producer node. All clustered HTTP requests should be sent through a producer node. The second server is configured on the server side as a consumer node with a server.tags field equal to worker.

Ingestion Steering

In the following example, you’ll notice this request differs from a typical file ingestion request. We have added the restClusterSettings object to the request and have specified that a server tagged as a worker should process this request through the use of nodeTags on the request.

curl --location --request POST 'http://localhost:8080/nuix-restful-service/svc/v1/cases/776a45b03d1841379b1dea524098a33b/evidence/file' \
--header 'nuix-auth-token: 9729a460-eda7-48dc-ba70-d12b3aae3c8d' \
--header 'Content-Type: application/json' \
--header 'Accept: application/json' \
--data-raw '{
   "processorSettings": {
   	"storeBinary": true
   },
   "target": {
    "path": "/Engines/9.0.0.171/doc"
  },
  "restClusterSettings": {
      "nodeTags": {
          "defaultTags": [ "worker" ]
      }
   }
}
'

{
    "functionKey": "62a120ce-bebe-492c-bbb5-fd68c99bb0db",
    "location": "http://localhost:8080/nuix-restful-service/svc/v1/asyncFunctions/62a120ce-bebe-492c-bbb5-fd68c99bb0db"
}

Ingestion Status

The functionKey field returned from the ingestion endpoint in the previous example can be polled for status.

curl --location --request GET 'http://localhost:8080/nuix-restful-service/svc/v1/asyncFunctions/62a120ce-bebe-492c-bbb5-fd68c99bb0db' \
--header 'nuix-auth-token: 9729a460-eda7-48dc-ba70-d12b3aae3c8d'

{
    "done": true,
    "cancelled": false,
    "result": true,
    "token": "eb4afd86-913f-4b52-8781-07d0523a5fd1",
    "functionKey": "62a120ce-bebe-492c-bbb5-fd68c99bb0db",
    "progress": 7581225,
    "total": 0,
    "percentComplete": null,
    "updatedOn": 1613072891973,
    "status": null,
    "statusId": null,
    "requestTime": 1613072849860,
    "startTime": 1613072850818,
    "finishTime": 1613072895964,
    "caseId": "776a45b03d1841379b1dea524098a33b",
    "caseName": "HelloElasticCase",
    "hasSuccessfullyCompleted": true,
    "friendlyName": "Evidence Ingestion Function",
    "caseLocation": "/Cases/HelloElasticCase",
    "requestor": "username",
    "action": "AsyncBulkIngestionFunction",
    "options": {
        "reloadQuery": null,
        "processorSettings": {
            "processText": null,
            "processLooseFileContents": null,
            "processForensicImages": null,
            "analysisLanguage": null,
            "stopWords": null,
            "stemming": null,
            "enableExactQueries": null,
            "extractNamedEntities": null,
            "extractNamedEntitiesFromText": null,
            "extractNamedEntitiesFromProperties": null,
            "extractNamedEntitiesFromTextStripped": null,
            "extractShingles": null,
            "processTextSummaries": null,
            "calculateSSDeepFuzzyHash": null,
            "detectFaces": null,
            "extractFromSlackSpace": null,
            "carveFileSystemUnallocatedSpace": null,
            "carveUnidentifiedData": null,
            "carvingBlockSize": null,
            "recoverDeletedFiles": null,
            "extractEndOfFileSlackSpace": null,
            "smartProcessRegistry": null,
            "identifyPhysicalFiles": null,
            "createThumbnails": null,
            "skinToneAnalysis": null,
            "calculateAuditedSize": null,
            "storeBinary": true,
            "maxStoredBinarySize": null,
            "maxDigestSize": null,
            "digests": [],
            "addBccToEmailDigests": null,
            "addCommunicationDateToEmailDigests": null,
            "reuseEvidenceStores": null,
            "processFamilyFields": null,
            "hideEmbeddedImmaterialData": null,
            "reportProcessingStatus": null,
            "workerItemCallback": null,
            "workerItemCallbacks": null
        },
        "evidence": [
            {
                "guid": null,
                "name": null,
                "customMetadata": null,
                "encoding": null,
                "custodian": null,
                "timeZone": null,
                "description": null,
                "locale": null,
                "files": [
                    {
                        "path": "/Engines/9.0.0.171/doc"
                    }
                ],
                "exchangeMailboxes": null,
                "s3Buckets": null,
                "sqlServers": null,
                "enterpriseVaults": null,
                "sharepointSites": null,
                "mailStores": null,
                "loadFiles": null,
                "centeraClusters": null,
                "splitFiles": null,
                "dropboxes": null,
                "sshServers": null
            }
        ],
        "localWorkerCount": 1,
        "repositories": [],
        "parallelProcessingSettings": {
            "workerCount": null,
            "workerMemory": null,
            "workerTemp": null,
            "brokerMemory": null,
            "workerBrokerAddress": null,
            "useRemoteWorkers": false,
            "embedBroker": true
        },
        "rescanEvidenceRepositories": false,
        "loadProcessingJob": {
            "casePath": "/Cases/HelloElasticCase",
            "jobGuid": "143cae10-1825-433b-8de4-71aa5ac0a722",
            "processingMode": "Load",
            "startDate": 1613072863682,
            "workerCount": 1,
            "finished": true,
            "paused": false,
            "masterAddress": "192.168.86.53",
            "bytesProcessed": 7581225,
            "itemsProcessed": 192,
            "jobSizeTotalBytes": 0
        }
    },
    "participatingInCaseFunctionQueue": true,
    "processedBy": "nuix-restful-server-2",
    "errorMsg": null
}

As you can see from polling the status, the response contains a processedBy field and this task was processed by nuix-restful-server-2.

Congratulations! You have successfully performed your first clustered ingestion and have distributed your workload.

Best Practice

In most cases, it is best to submit requests to the cluster and allow the cluster to decide where the task should be processed. By default, each node in the cluster attempts to process as many tasks from the task queue as it can. Following this best practice provides a natural load balancing effect.

There are times when you may want to reserve dedicated hardware for a task. One example might be designating better equipped hardware for OCR operations. In this case you might set the server.tags property on that server to ocr and steer PUT /cases/{caseId}/items/ocr requests. See the following example for detailed request and response information.

curl --location --request PUT 'http://localhost:8080/nuix-restful-service/svc/v1/cases/62a120ce-bebe-492c-bbb5-fd68c99bb0db/items/ocr' \
--header 'nuix-auth-token: 9729a460-eda7-48dc-ba70-d12b3aae3c8d'
--header 'Content-Type: application/json' \
--header 'Accept: application/json' \
--data-raw '{
  "parallelProcessingSettings": {
    "embedBroker": true,
    "workerCount": 4,
    "workerMemory": 2048
  },
  "query": "guid:175d00fe-2da7-4adb-8486-98dcfa235d16",
    "restClusterSettings": {
      "nodeTags": {
          "defaultTags": [ "ocr" ]
      }
   }
}'

{
  "functionKey": "e2815926-966d-45b7-8974-de7f4fd05e5c", 
  "location": "http://localhost:8080/nuix-restful-service/svc/v1/asyncFunctions/e2815926-966d-45b7-8974-de7f4fd05e5c"
}

curl --location --request GET 'http://localhost:8080/nuix-restful-service/svc/v1/asyncFunctions/e2815926-966d-45b7-8974-de7f4fd05e5c' \
--header 'nuix-auth-token: 9729a460-eda7-48dc-ba70-d12b3aae3c8d'

{
    "done": true,
    "cancelled": false,
    "result": {
        "total": 1,
        "failure": 0
    },
    "token": "9729a460-eda7-48dc-ba70-d12b3aae3c8d",
    "functionKey": "e2815926-966d-45b7-8974-de7f4fd05e5c",
    "progress": 1,
    "total": 1,
    "percentComplete": 100.0000,
    "updatedOn": 1634322969490,
    "status": null,
    "statusId": null,
    "requestTime": 1634322821242,
    "startTime": 1634322821494,
    "finishTime": 1634322973313,
    "caseId": "4b32357a3894481d804266128fa93661",
    "caseName": "SearchBestPractice",
    "hasSuccessfullyCompleted": true,
    "friendlyName": "OCR Function",
    "caseLocation": "/Cases/SearchBestPractice",
    "requestor": "nuixadmin",
    "licenseShortName": "enterprise-workstation",
    "action": "AsyncOcrFunction",
    "options": {
        "ocrOptions": {
            "regeneratePdfs": null,
            "updatePdf": null,
            "updateText": null,
            "textModification": null,
            "quality": null,
            "rotation": null,
            "deskew": null,
            "clearOcrCache": null,
            "outputDirectory": null,
            "languages": [
                "ENGLISH"
            ],
            "timeout": null,
            "updateDuplicates": null,
            "ocrProfileName": null
        },
        "localWorkerCount": 1,
        "parallelProcessingSettings": {
            "workerCount": 4,
            "workerMemory": 2048,
            "workerTemp": null,
            "brokerMemory": null,
            "workerBrokerAddress": null,
            "useRemoteWorkers": false,
            "embedBroker": true
        },
        "query": "guid:175d00fe-2da7-4adb-8486-98dcfa235d16",
        "ocrProfile": null,
        "ocrImagingSettings": null,
        "imagingProfile": null,
        "tags": null
    },
    "participatingInCaseFunctionQueue": true,
    "processedBy": "nuix-restful-server-2",
    "errorMsg": null
}