EVOLUTION-MANAGER

Edit File: source_remote.py

# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Communicating tracebacks and source code with debug server."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import socket

import grpc

from tensorflow.core.debug import debug_service_pb2
from tensorflow.core.protobuf import debug_pb2
from tensorflow.python.debug.lib import common
from tensorflow.python.debug.lib import debug_service_pb2_grpc
from tensorflow.python.debug.lib import source_utils
from tensorflow.python.platform import gfile
from tensorflow.python.profiler import tfprof_logger

def _load_debugged_source_file(file_path, source_file_proto):
  file_stat = gfile.Stat(file_path)
  source_file_proto.host = socket.gethostname()
  source_file_proto.file_path = file_path
  source_file_proto.last_modified = file_stat.mtime_nsec
  source_file_proto.bytes = file_stat.length
  try:
    with gfile.Open(file_path, "r") as f:
      source_file_proto.lines.extend(f.read().splitlines())
  except IOError:
    pass

def _string_to_id(string, string_to_id):
  if string not in string_to_id:
    string_to_id[string] = len(string_to_id)
  return string_to_id[string]

def _format_origin_stack(origin_stack, call_traceback_proto):
  """Format a traceback stack for a `CallTraceback` proto.

Args:
    origin_stack: The stack list as returned by `traceback.extract_stack()`.
    call_traceback_proto: A `CallTraceback` proto whose fields are to be
      populated.
  """
  string_to_id = {}
  string_to_id[None] = 0
  for frame in origin_stack:
    file_path, lineno, func_name, line_text = frame
    call_traceback_proto.origin_stack.traces.add(
        file_id=_string_to_id(file_path, string_to_id),
        lineno=lineno,
        function_id=_string_to_id(func_name, string_to_id),
        line_id=_string_to_id(line_text, string_to_id))

id_to_string = call_traceback_proto.origin_id_to_string
  for key, value in string_to_id.items():
    id_to_string[value] = key if key is not None else ""

def _source_file_paths_outside_tensorflow_py_library(code_defs, id_to_string):
  """Extract source file paths outside TensorFlow Python library.

Args:
    code_defs: An iterable of `CodeDef` protos, i.e., an iterable of stack
      traces.
    id_to_string: A proto map from integer ids to strings.

Returns:
    An iterable of source file paths outside the TensorFlow Python library.
  """
  file_ids = set()
  for code_def in code_defs:
    for trace in code_def.traces:
      file_ids.add(trace.file_id)
  non_tf_files = (id_to_string[file_id] for file_id in file_ids)
  non_tf_files = (
      f for f in non_tf_files
      if not source_utils.guess_is_tensorflow_py_library(f) and gfile.Exists(f))
  return non_tf_files

def _send_call_tracebacks(destinations,
                          origin_stack,
                          is_eager_execution=False,
                          call_key=None,
                          graph=None,
                          send_source=True):
  """Send the tracebacks of a TensorFlow execution call.

To gRPC debug server(s). This applies to graph execution (`tf.Session.run()`)
  calls and eager execution calls.

If `send_source`, also sends the underlying source files outside the
  TensorFlow library.

Args:
    destinations: gRPC destination addresses, a `str` or a `list` of `str`s,
      e.g., "localhost:4242". If a `list`, gRPC requests containing the same
      `CallTraceback` proto payload will be sent to all the destinations.
    origin_stack: The traceback stack for the origin of the execution call. For
      graph execution, this is the traceback of the `tf.Session.run()`
      invocation. For eager execution, this is the traceback of the Python
      line that executes the eager operation.
    is_eager_execution: (`bool`) whether an eager execution call (i.e., not a
      `tf.Session.run` or derived methods) is being sent.
    call_key: The key of the execution call, as a string. For graph execution,
      this is a string describing the feeds, fetches (and targets) names of the
      `tf.Session.run` call. For eager execution, this is ignored.
    graph: A Python `tf.Graph` object (i.e., *not* a `tf.compat.v1.GraphDef`),
      which contains op tracebacks, if applicable.
    send_source: Whether the source files involved in the op tracebacks but
      outside the TensorFlow library are to be sent.
  """
  if not isinstance(destinations, list):
    destinations = [destinations]
  # Strip grpc:// prefix, if any is present.
  destinations = [
      dest[len(common.GRPC_URL_PREFIX):]
      if dest.startswith(common.GRPC_URL_PREFIX) else dest
      for dest in destinations]

call_type = (debug_service_pb2.CallTraceback.EAGER_EXECUTION
               if is_eager_execution
               else debug_service_pb2.CallTraceback.GRAPH_EXECUTION)
  graph_traceback = tfprof_logger.merge_default_with_oplog(
      graph, add_trainable_var=False) if graph else None
  call_traceback = debug_service_pb2.CallTraceback(
      call_type=call_type, call_key=call_key, graph_traceback=graph_traceback,
      graph_version=graph.version if graph else None)

_format_origin_stack(origin_stack, call_traceback)

if send_source:
    source_file_paths = set()
    source_file_paths.update(_source_file_paths_outside_tensorflow_py_library(
        (log_entry.code_def for log_entry
         in call_traceback.graph_traceback.log_entries),
        call_traceback.graph_traceback.id_to_string))
    source_file_paths.update(_source_file_paths_outside_tensorflow_py_library(
        [call_traceback.origin_stack], call_traceback.origin_id_to_string))

debugged_source_files = []
    for file_path in source_file_paths:
      source_files = debug_pb2.DebuggedSourceFiles()
      _load_debugged_source_file(
          file_path, source_files.source_files.add())
      debugged_source_files.append(source_files)

for destination in destinations:
    no_max_message_sizes = [("grpc.max_receive_message_length", -1),
                            ("grpc.max_send_message_length", -1)]
    channel = grpc.insecure_channel(destination, options=no_max_message_sizes)
    stub = debug_service_pb2_grpc.EventListenerStub(channel)
    stub.SendTracebacks(call_traceback)
    if send_source:
      for source_files in debugged_source_files:
        stub.SendSourceFiles(source_files)

def send_graph_tracebacks(destinations,
                          run_key,
                          origin_stack,
                          graph,
                          send_source=True):
  """Send the tracebacks of a graph execution call to debug server(s).

Args:
    destinations: gRPC destination addresses, a `str` or a `list` of `str`s,
      e.g., "localhost:4242". If a `list`, gRPC requests containing the same
      `CallTraceback` proto payload will be sent to all the destinations.
    run_key: A string describing the feeds, fetches (and targets) names of the
      `tf.Session.run` call.
    origin_stack: The traceback of the `tf.Session.run()` invocation.
    graph: A Python `tf.Graph` object (i.e., *not* a `tf.compat.v1.GraphDef`),
      which contains op tracebacks.
    send_source: Whether the source files involved in the op tracebacks but
      outside the TensorFlow library are to be sent.
  """
  _send_call_tracebacks(
      destinations, origin_stack, is_eager_execution=False, call_key=run_key,
      graph=graph, send_source=send_source)

def send_eager_tracebacks(destinations,
                          origin_stack,
                          send_source=True):
  """Send the tracebacks of an eager execution call to debug server(s).

Args:
    destinations: gRPC destination addresses, a `str` or a `list` of `str`s,
      e.g., "localhost:4242". If a `list`, gRPC requests containing the same
    origin_stack: The traceback of the eager operation invocation.
    send_source: Whether the source files involved in the op tracebacks but
      outside the TensorFlow library are to be sent.
  """
  _send_call_tracebacks(
      destinations, origin_stack, is_eager_execution=True,
      send_source=send_source)