app_transformation.py

# You can find this code for Chainlit python streaming here (https://docs.chainlit.io/concepts/streaming/python)

# OpenAI Chat completion
import os
from openai import AsyncOpenAI  # importing openai for API usage
import chainlit as cl  # importing chainlit for our app
from chainlit.prompt import Prompt, PromptMessage  # importing prompt tools
from chainlit.playground.providers import ChatOpenAI  # importing ChatOpenAI tools
from dotenv import load_dotenv

load_dotenv()


# ChatOpenAI Templates
system_template = """You are a cybersecurity expert specialized in log analysis and data normalization,/n
helping security teams to map security log fields to Google Chronicle's Unified Data Model (UDM).

Please follow these steps:
1. Use web search to consult the latest log documentation for the product provided and Google Chronicle UDM schema documentation.
2. Map each product log field to its corresponding UDM field.
3. For fields that don't have a direct match in UDM, place them into custom fields. 
4. make sure the each mapped field including the custome fields are unique and accurate
5. Organize the mapping into a structured table format.
Remember to explain your reasoning for each field's mapping based on your findings from the product documentation.
Think through your response step by step, and include references to the documentation you consulted during the process.
You always provide detailed, accurate, and structured responses in a professional tone, focusing on precision.
"""

user_template = """Here is a sample log:

{input}

Please follow these steps:
1. Use web search to consult the latest documentation for this product/log source and Google Chronicle UDM schema documentation.
2. Map each log field to its corresponding UDM field.
3. For fields that don't have a direct match in UDM, place them into custom fields.
4. make sure the each mapped field including the custome fields are unique and accurate
5. Organize the mapping into a structured table format.

Here is an example of how to map FortiGate fields to UDM fields.. 
Consider this example and related search into relevant URLs to identify best and most accurate mapping for other log sources against UB
Log_Attribute	Simpliefied_Naming_for_Attribute	UBM_Attribute
devname	devicehostname	intermediary.hostname
devid	deviceexternalid	intermediary.asset.hardware.serial_number
ap	customstring62	target.application
dstport	destinationport	target.port
tranport	destinationtranslatedport	target.nat_port
policytype	customstring55	security_result.rule_type
TransportProtocol	transportprotocol	network.ip_protocol
srcip	sourceaddress	principal.ip
dstmac	destinationmacaddress	target.mac
eventtype	customstring28	security_result.rule_type
level	deviceseverity	security_result.severity_details
filename	filename	target.file.full_path
action	deviceaction	security_result.action_details
ipaddr	destinationtranslatedaddress	target.nat_ip
srcuuid	flowsiemid	principal.asset.product_object_id
policyid	customnumber22	additional.fields.policyid
srcport	sourceport	principal.port
service	applicationprotocol	network.application_protocol_version
to	emailrecipient	network.email.to
appid	customstring34	security_result.rule_id
proto	customnumber6	network.ip_protocol
ref	devicecustomstring6	network.http.referral_url
method	requestmethod	network.http.method
transport	translatedport	src.nat_port
srcname	sourcehostname	principal.user.first_name
duration	transactionstring4	network.session_duration.seconds
hostname	destinationhostname	target.hostname
Header2	deviceaddress	intermediary.hostname
url	requesturl	target.url
domain	emailsenderdomain	network.dns_domain
agent	requestclientapplication	network.http.user_agent
rcvdpkt	transactionnumber4	additional.fields.rcvdpkt
catdesc	deviceeventcategory	security_result.category_details
sentbyte	bytesout	network.sent_bytes
sentpkt	transactionnumber3	additional.fields.sentpkt
msg	message	metadata.description
source_mac	sourcemacaddress	principal.mac
subject	emailsubject	network.email.subject
from	emailsender	network.email.from
dstip	destinationaddress	target.ip
logid	transactionstring6	metadata.product_log_id
rcvdbyte	bytesin	network.received_bytes
type	customstring1	metadata.product_event_type
user	accountname	principal.user.user_display_name
sessionid	sessionid	network.session_id
dstuser	destinationusername	target.user.userid

References:
FortiGate Log Documentation, check all links in this URL: https://docs.cyderes.cloud/parser-knowledge-base/fortinet_firewall/
Google Chronicle UDM Documentation:https://cloud.google.com/chronicle/docs/reference/udm-field-list

The official documentation of Chronicle UDM, explaining the different fields and structures available for mapping.

Think through your response step by step, and include references to the documentation you consulted during the process.
"""


@cl.on_chat_start  # marks a function that will be executed at the start of a user session
async def start_chat():
    settings = {
        "model": "gpt-3.5-turbo",
        "temperature": 0,
        "max_tokens": 500,
        "top_p": 1,
        "frequency_penalty": 0,
        "presence_penalty": 0,
    }

    cl.user_session.set("settings", settings)


@cl.on_message  # marks a function that should be run each time the chatbot receives a message from a user
async def main(message: cl.Message):
    settings = cl.user_session.get("settings")

    client = AsyncOpenAI()

    print(message.content)

    prompt = Prompt(
        provider=ChatOpenAI.id,
        messages=[
            PromptMessage(
                role="system",
                template=system_template,
                formatted=system_template,
            ),
            PromptMessage(
                role="user",
                template=user_template,
                formatted=user_template.format(input=message.content),
            ),
        ],
        inputs={"input": message.content},
        settings=settings,
    )

    print([m.to_openai() for m in prompt.messages])

    msg = cl.Message(content="")

    # Call OpenAI
    async for stream_resp in await client.chat.completions.create(
        messages=[m.to_openai() for m in prompt.messages], stream=True, **settings
    ):
        token = stream_resp.choices[0].delta.content
        if not token:
            token = ""
        await msg.stream_token(token)

    # Update the prompt object with the completion
    prompt.completion = msg.content
    msg.prompt = prompt

    # Send and close the message stream
    await msg.send()