LocalInference

Attributes

attributenamestr

= 'local'

attributeConfigtype

= LocalInferenceConfig

attributetokenizer

= AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)

attributemodel

= AutoModelForCausalLM.from_pretrained(model_name, dtype=torch_dtype, trust_remote_code=True, device_map=(device if device != 'auto' else 'auto'))

func__init__(self, *, model_name, adapter_path=None, dtype='bfloat16', device='auto') -> None

paramself

parammodel_namestr

paramadapter_pathstr | None

= None

paramdtypestr

= 'bfloat16'

paramdevicestr

= 'auto'

Returns

None

funcgenerate(self, *, prompt, max_tokens=256, temperature=0.0, stop=None) -> str

paramself

parampromptstr

parammax_tokensint

= 256

paramtemperaturefloat

= 0.0

paramstoplist[str] | None

= None

Returns

str