0.0.0+develop

flytekitplugins.inference.nim.serve

Directory

Classes

Class Description
NIM
NIMSecrets .

flytekitplugins.inference.nim.serve.NIM

class NIM(
    secrets: flytekitplugins.inference.nim.serve.NIMSecrets,
    image: str,
    health_endpoint: str,
    port: int,
    cpu: int,
    gpu: int,
    mem: str,
    shm_size: str,
    env: typing.Optional[dict[str, str]],
    hf_repo_ids: typing.Optional[list[str]],
    lora_adapter_mem: typing.Optional[str],
)

Initialize NIM class for managing a Kubernetes pod template.

Parameter Type
secrets flytekitplugins.inference.nim.serve.NIMSecrets
image str
health_endpoint str
port int
cpu int
gpu int
mem str
shm_size str
env typing.Optional[dict[str, str]]
hf_repo_ids typing.Optional[list[str]]
lora_adapter_mem typing.Optional[str]

Methods

Method Description
setup_nim_pod_template()

setup_nim_pod_template()

def setup_nim_pod_template()

Properties

Property Type Description
base_url
pod_template

flytekitplugins.inference.nim.serve.NIMSecrets

class NIMSecrets(
    ngc_image_secret: str,
    ngc_secret_key: str,
    secrets_prefix: str,
    ngc_secret_group: typing.Optional[str],
    hf_token_group: typing.Optional[str],
    hf_token_key: typing.Optional[str],
)
Parameter Type
ngc_image_secret str
ngc_secret_key str
secrets_prefix str
ngc_secret_group typing.Optional[str]
hf_token_group typing.Optional[str]
hf_token_key typing.Optional[str]