// This proto is used for inference-specific analysis.
syntax = "proto2";

package tensorflow.profiler;

message TensorEventDetail {
  // The index of the tensor pattern in TensorPatternDatabase.
  optional int32 tensor_pattern_index = 1;

  // The owner of this TensorEventDetail.
  enum TensorEventOwner {
    // Unknown. This should not happen in production code.
    UNKNOWN = 0;

    // Owned by the request.
    REQUEST = 1;

    // Owned by the batch.
    BATCH = 2;
  }

  // If batching is enabled, the TensorEventDetails in BatchDetail will have
  // owner = BATCH, and they are counted when calculating statistics like the
  // number of occurrence for each tensor pattern. The TensorEventDetails in
  // RequestDetail will have owner = BATCH, which means the tensor events
  // actually happen in the batch, and they are not counted when calculating
  // various statistics.
  // If batching is not enabled, the TensorEventDetail will only appear in
  // RequestDetail and the owner will only be REQUEST.
  optional TensorEventOwner owner = 2;

  // Total time in picosecs spent on linearize and delinearize tensors.
  optional uint64 linearize_delinearize_time_ps = 3;
}

// Detail of a user facing request.
// Next ID: 22
message RequestDetail {
  // Request id.
  optional int64 request_id = 10 [default = -1];

  // An index to the model_id inside InferenceStats below. Storing index
  // instead of string to save space. It will be -1 if the model id is not
  // given.
  optional int32 model_id_index = 8 [default = -1];

  // Start-time of the request in picosecs.
  optional uint64 start_time_ps = 1 [default = 0];

  // End-time of the request in picosecs.
  optional uint64 end_time_ps = 2 [default = 0];

  // Total time in picosecs in this request spent on device.
  optional uint64 device_time_ps = 7 [default = 0];

  // Total time in picosecs in this request spent on writes to device.
  optional uint64 write_to_device_time_ps = 5 [default = 0];

  // Total time in picosecs in this request spent on reads from device.
  optional uint64 read_from_device_time_ps = 6 [default = 0];

  // If this inference request is running in batching mode, record the latency
  // between a request is scheduled and is processed in a batch. Otherwise, it
  // will always be 0.
  optional uint64 batching_request_delay_ps = 12 [default = 0];

  // Batch ids related to this request.
  repeated int64 related_batch_ids = 11;

  // If this inference request is running in batching mode, record the size of
  // the request. Otherwise, it will always be 0.
  optional int32 batching_request_size = 13;

  // Detailed breakdown for host side activities of a request.
  // Total time in picosecs spent on host preprocessing.
  optional uint64 host_preprocessing_ps = 14;

  // Total time in picosecs spent on host batch formation.
  optional uint64 host_batch_formation_ps = 15;

  // Total time in picosecs spent on host runtime.
  optional uint64 host_runtime_ps = 16;

  // Total time in picosecs spent on host postprocessing.
  optional uint64 host_postprocessing_ps = 17;

  // Tensor event details.
  // One request can have multiple TensorEventDetails because it might be
  // split into multiple batches for execution.
  repeated TensorEventDetail tensor_event_details = 18;

  // Host index for this request.
  optional int32 host_id = 19;

  // Percentile of this request in all requests in the profile duration.
  optional double percentile = 20;

  // The time no event associated with. It could be that the machine was idle or
  // executing some events which were not traced.
  optional double idle_time_ps = 21;

  // Were device_start_time_ps, device_end_time_ps, session_id
  reserved 3, 4, 9;
}

// Detail of a batch.
// Next ID: 13
message BatchDetail {
  // Batch id.
  optional int64 batch_id = 1 [default = -1];

  // Start time of the batch in picosecs.
  optional uint64 start_time_ps = 2 [default = 0];

  // End time of the batch in picosecs.
  optional uint64 end_time_ps = 3 [default = 0];

  // The latency between "start time of the first request in this batch", and
  // the time this batch is processed.
  optional uint64 batch_delay_ps = 5 [default = 0];

  // Request ids related to this batch.
  repeated int64 related_request_ids = 4;

  // Size of padding.
  optional int32 padding_amount = 6;

  // Size of a batch after padding.
  optional int32 batch_size_after_padding = 7;

  // Model ID of this batch. This is the same model_id as any of the request in
  // this batch. All the requests from the same batch must share the same
  // model_id.
  optional int32 model_id_index = 8;

  // Tensor event details.
  optional TensorEventDetail tensor_event_detail = 9;

  // Host index for this batch.
  optional int32 host_id = 10;

  // Percentile of this batch in all batches in the profile duration.
  optional double percentile = 11;

  // Total time in picosecs in this batch spent on device.
  optional uint64 device_time_ps = 12 [default = 0];
}

// Per-host data for inference analysis.
message PerHostInferenceStats {
  // A list of requests selected for inference analysis on this host.
  // This list is in ascending order of the request duration.
  repeated RequestDetail request_details = 3;

  // A list of batches selected for inference analysis on this host.
  // This list is in ascending order of the batch duration.
  repeated BatchDetail batch_details = 5;

  reserved 1, 2, 4, 6;

  // were session_run_times, sessions_per_second, requests_per_second,
  // batches_per_second.
}

// Per-model aggregated result of tensor transfer.
message TensorTransferAggregatedResult {
  message TensorPatternResult {
    // The index of the tensor pattern in TensorPatternDatabase.
    optional int32 tensor_pattern_index = 1;

    // The number of occurrence of this tensor pattern in this model.
    optional uint64 count = 2;

    message PercentileTime {
      optional double percentile = 1;
      optional uint64 time_ps = 2;
    }

    // The percentiles of the linearize and delinearize time of this tensor
    // pattern in this model.
    repeated PercentileTime linearize_delinearize_percentile_time = 3;
  }

  repeated TensorPatternResult tensor_pattern_results = 1;
}

// Aggregated result per batch size.
message PerBatchSizeAggregatedResult {
  optional int32 batch_size = 1;
  optional RequestDetail aggregated_request_result = 2;
  optional BatchDetail aggregated_batch_result = 3;
  optional double request_throughput = 4;
  optional double batch_throughput = 5;
}

// Per-model data for inference analysis.
message PerModelInferenceStats {
  // A list of requests selected for inference analysis on this model.
  // This list is in ascending order of the request duration.
  repeated RequestDetail request_details = 1;

  // Aggregated result from all the <request_details>.
  optional RequestDetail aggregated_request_detail = 8;

  // Inference requests per second for this model.
  optional double request_throughput = 2;

  // Average latency in microseconds of the requests in this model.
  optional double request_average_latency_us = 3;

  // A list of batches selected for inference analysis on this model.
  // This list is in ascending order of the batch duration.
  repeated BatchDetail batch_details = 4;

  // Aggregated result from all the <batch_details>.
  optional BatchDetail aggregated_batch_detail = 9;

  // Batches per second for this model.
  optional double batch_throughput = 5;

  // Average latency in microseconds of the batches in this model.
  optional double batch_average_latency_us = 6;

  // The aggregated result of tensor transfer in this model.
  optional TensorTransferAggregatedResult tensor_transfer_aggregated_result = 7;

  // Aggregated result per batch size.
  repeated PerBatchSizeAggregatedResult per_batch_size_aggregated_result = 10;
}

// Batching parameters collected from TFstreamz.
message BatchingParameters {
  // Number of batch threads.
  optional int64 num_batch_threads = 1;

  // How long a request can wait before being processed by a batch.
  optional int64 batch_timeout_micros = 2;

  // Maximum size of a batch.
  optional int64 max_batch_size = 3;

  // Maximum number of enqueued batches.
  optional int64 max_enqueued_batches = 4;

  // Sizes that are allowed to form a batch. A list of integers separated by ","
  optional string allowed_batch_sizes = 5;
}

// Model ID database. Unknown model id will be "" and won't be stored here. So
// if model id is not found in the TF-session metadata, ModelIdDatabase will be
// empty.
message ModelIdDatabase {
  // Array of model ids.
  repeated string ids = 1;

  // Map from id to index.
  map<string, int32> id_to_index = 2;

  // Map from id to batching parameters.
  map<string, BatchingParameters> id_to_batching_params = 3;
}

// Tensor pattern database for all the tensor patterns that occurred during the
// profiling window.
message TensorPatternDatabase {
  // A tensor pattern is the string concatenation of all the linearize and
  // delinearize events in an inference request. Each event records the tensor
  // shape, data type and the layout on device.
  repeated string tensor_pattern = 1;
}

// Proto consumed by inference analysis.
message InferenceStats {
  // Map from host-id to the InferenceStats for that host.
  map<int32 /* host-id */, PerHostInferenceStats> inference_stats_per_host = 3;

  // Map from model-id to the InferenceStats for that model.
  map<int32 /* model-id */, PerModelInferenceStats> inference_stats_per_model =
      5;

  // A database of model ids.
  optional ModelIdDatabase model_id_db = 4;

  // A database of tensor patterns.
  optional TensorPatternDatabase tensor_pattern_db = 6;

  optional SampledInferenceStatsProto sampled_inference_stats = 7;

  reserved 1, 2;  // were processing_stats, session_run_times
}

message SampledPerModelInferenceStatsProto {
  repeated RequestDetail sampled_requests = 1;
  repeated BatchDetail sampled_batches = 2;
}

message SampledInferenceStatsProto {
  // Map from model index to the Sampled Stats.
  map<int32 /* host-id */, SampledPerModelInferenceStatsProto>
      sampled_inference_stats_per_model = 1;
}
