Copilot: Azure dataloader enhancements

Please note this regards the dataloader for Azure mainly

• Question: Is the loading of 12.4 docs intentional?

• defect: At least routine for the docs_spotfire_analyst_um is called twice

• enhancement: allow to specify an alternative index for spotfiredocs, e.g. when a new dataloader is available we want to be able to test the loading mechanism

• enhancement: allow specifying chunk_size and chunk_overlap

• enhancement: allow specifying the load of core and/or additional files if bullet 1 is intentional

• enhancement: please specify the index sizes approximately in the installation and/or instructions so that one knows the capacity required on Azure, e.g. free tier can barely hold analyst and copilot docs

@app.post("/load_spotfire_docs/",

          summary="Load Spotfire documents",

          response_model=LoadResponse)

def handle_request(current_user: Annotated[authentication.User, Depends(authentication.get_current_active_user)],

    load_request: LoadSpotfireRequest

    ) -> any:

    '''

    Load Spotfire documents to Azure Cognitive Search service.

    '''

    logger.info("Processing load request.")



    # Load docs

    try:

        if load_request.create_index:

            delete_search_index("spotfiredocs")

            create_search_index("spotfiredocs")

        elif not search_index_exists("spotfiredocs"):

            raise HTTPException(

                status_code=404,

                detail="Azure Cognitive Search index 'spotfiredocs' was not found. Set create_index=true to create it first.",

            )



        # Load documents

        loader = DirectoryLoader(

            "./docs/docs_spotfire_analyst_um", glob="**/*.pdf", loader_cls=PyPDFLoader

        )

        docs = loader.load()

        count_before = get_index_document_count("spotfiredocs")

        indexed_docs_count = process_docs(docs, "spotfiredocs")



        loader = DirectoryLoader(

            "./docs/docs_spotfire_server", glob="**/*.pdf", loader_cls=PyPDFLoader

        )

        docs = loader.load()

        indexed_docs_count += process_docs(docs, "spotfiredocs")



        loader = DirectoryLoader(

            "./docs/docs_spotfire_mods", glob="**/*.pdf", loader_cls=PyPDFLoader

        )

        docs = loader.load()

        indexed_docs_count += process_docs(docs, "spotfiredocs")



        loader = DirectoryLoader(

            "./docs/docs_spotfire_analyst_um", glob="**/*.pdf", loader_cls=PyPDFLoader

        )

        docs = loader.load()

        indexed_docs_count += process_docs(docs, "spotfiredocs")

        current_count = get_index_document_count_with_retry("spotfiredocs")

        count_delta = current_count - count_before

        logger.info(

            "Index 'spotfiredocs' currently reports %d chunk(s) (delta %+d). Submitted %d chunk(s) in this request. Count may lag briefly due Azure indexing latency.",

            current_count,

            count_delta,

            indexed_docs_count,

        )



    except HTTPException:

        raise

    except Exception as e:

        logger.exception("Error while loading data")

        raise _internal_error_response()



    return {"result": "Success"}