# model parameters
W = tf.Variable(0.3,  tf.float32)
b = tf.Variable(-0.3, tf.float32)
# model inputs & outputs
x = tf.placeholder(tf.float32)
y = tf.placeholder(tf.float32)
# the model
out = W * x + b
# loss function
loss = tf.reduce_sum(tf.square(out - y))
# optimizer
optimizer = tf.train.GradientDescentOptimizer(0.001)
train = optimizer.minimize(loss)
# training data
x_train = np.random.random_sample((100,)).astype(np.float32)
y_train = np.random.random_sample((100,)).astype(np.float32)
# training
init = tf.global_variables_initializer()
with tf.Session() as sess:
    sess.run(init)
    for i in range(1000):
        sess.run(train, {x:x_train, y:y_train})
        current_loss = sess.run(loss, {x:x_train, y:y_train})
        print("iter step %d training loss %f" % (i, current_loss))
    print(sess.run(W))
    print(sess.run(b))

# 返回 _NumericColumn
price = numeric_column('price')
keywords_embedded = embedding_column(categorical_column_with_hash_bucket("keywords", 10K), dimensions=16)
columns = [price, keywords_embedded, ...]
features = tf.parse_example(..., features=make_parse_example_spec(columns))
dense_tensor = input_layer(features, columns)
for units in [128, 64, 32]:
    dense_tensor = tf.layers.dense(dense_tensor, units, tf.nn.relu)
prediction = tf.layers.dense(dense_tensor, 1)  # 也是output

# 
# Returns a dense `Tensor` as input layer based on given `feature_columns`
def input_layer(features,feature_columns,...):
    #  features: A mapping from key to tensors. `_FeatureColumn`s look up via these
    #  keys. For example `numeric_column('price')` will look at 'price' key in
    #  this dict. Values can be a `SparseTensor` or a `Tensor` depends on
    #  corresponding `_FeatureColumn`.
    ...
    builder = _LazyBuilder(features)
    output_tensors = []
    for column in sorted(feature_columns, key=lambda x: x.name):
        tensor = column._get_dense_tensor(builder,...)
        tensor = array_ops.reshape(tensor, shape=(batch_size, num_elements))
        output_tensors.append(tensor)
    return array_ops.concat(output_tensors, 1)
def linear_model(features,feature_columns,...)
def _transform_features(features, feature_columns,...)
def make_parse_example_spec(feature_columns)
# 构建FeatureColumn
def embedding_column(categorical_column, dimension,...):
    ...
    return _EmbeddingColumn(categorical_column=categorical_column,...)
def numeric_column(key,...):
    ...
    return _NumericColumn(key,shape=shape,...)
def bucketized_column(source_column, boundaries, ...)
def categorical_column_with_hash_bucket(key,...)
def categorical_column_with_vocabulary_file(key, vocabulary_file, vocabulary_size,...)
def crossed_column(keys, hash_bucket_size, ...)
...

class _FeatureColumn(object):
	def name(self):
	def _transform_feature(self, inputs):       # inputs 是一个_LazyBuilder，返回一个tensor
class _DenseColumn(_FeatureColumn):
	def _variable_shape(self):
	def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None):
class _CategoricalColumn(_FeatureColumn):
	IdWeightPair = collections.namedtuple(  'IdWeightPair', ['id_tensor', 'weight_tensor'])
    def _num_buckets(self):
    def _get_sparse_tensors(self,inputs,...):     # Returns an IdWeightPair

class _LazyBuilder(object):
    def __init__(self, features):
    	self._features = features.copy()  # 真正的input(dict)
    	self._feature_tensors = {}        # 缓存作用
   	def get(self, key):
   		if key in self._feature_tensors:
            # FeatureColumn is already transformed or converted.
            return self._feature_tensors[key]
	    if key in self._features:
	    	feature_tensor = self._get_raw_feature_as_tensor(key)
		    self._feature_tensors[key] = feature_tensor
		    return feature_tensor
        ... # 此时key 是一个 _FeatureColumn
		column = key
		transformed = column._transform_feature(self)
		self._feature_tensors[column] = transformed
        return transformed

class _NumericColumn(_DenseColumn,
                     collections.namedtuple('_NumericColumn', [
                         'key', 'shape', 'default_value', 'dtype',
                         'normalizer_fn'
                     ])):
    def _transform_feature(self, inputs):
        input_tensor = inputs.get(self.key)
        # 将input_tensor 转换为float 数据
        return math_ops.to_float(input_tensor)

class _EmbeddingColumn(
    _DenseColumn,
    collections.namedtuple('_EmbeddingColumn', (
        'categorical_column', 'dimension', 'combiner', 'initializer',
        'ckpt_to_load_from', 'tensor_name_in_ckpt', 'max_norm', 'trainable'
    ))):
    def _transform_feature(self, inputs):
        return inputs.get(self.categorical_column)
    def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None):
        # Get sparse IDs and weights. 比如构建 one-hot 向量
        sparse_tensors = self.categorical_column._get_sparse_tensors(  # pylint: disable=protected-access
            inputs, weight_collections=weight_collections, trainable=trainable)
        sparse_ids = sparse_tensors.id_tensor
        sparse_weights = sparse_tensors.weight_tensor
        # 新建一个 embedding_weights 权重矩阵 
        embedding_weights = variable_scope.get_variable(name='embedding_weights',...)
        ...
        # 根据 sparse_ids/sparse_weights 从权重 矩阵中检索 降维后的矩阵
        return _safe_embedding_lookup_sparse(embedding_weights=embedding_weights,sparse_ids=sparse_ids,sparse_weights=sparse_weights,...)
def _safe_embedding_lookup_sparse(embedding_weights,sparse_ids,sparse_weights=None,...):
    ... # 对输入参数 进行处理
    result = embedding_ops.embedding_lookup_sparse(embedding_weights,sparse_ids,sparse_weights,...)
    final_result = array_ops.reshape(result,...)
    return final_result

# tensorflow/python/feature_column/feature_column_v2.py
class StateManager(object):
	def create_variable(self,feature_column,name,...)
	def add_variable(self, feature_column, var):
	def get_variable(self, feature_column, name):
    def add_resource(self, feature_column, name, resource):
	def has_resource(self, feature_column, name):
	def get_resource(self, feature_column, name):
class _StateManagerImpl(StateManager):
    def __init__(self, layer, trainable):
        self._trainable = trainable
        self._layer = layer
        if self._layer is not None and not hasattr(self._layer, '_resources'):
            self._layer._resources = data_structures.Mapping()  # pylint: disable=protected-access
        self._cols_to_vars_map = collections.defaultdict(lambda: {})
        self._cols_to_resources_map = collections.defaultdict(lambda: {})

# tensorflow/tensorflow/python/ops/variables.py
class Variable(object):
    def __init__(self, initial_value=None, trainable=True,collections=None, name=None, dtype=None):
        ...
        self._init_from_args(initial_value=initial_value,trainable=trainable,...)
    def _init_from_args(self,initial_value=None,...):
        ...
        # initial_value: 初始值，为一个tensor，或者可以被包装为tensor的值
        # trainable：是否可以训练，如果为false，则训练时不会改变
        # collections：变量要加入哪个集合中，有全局变量集合、本地变量集合、可训练变量集合等。默认加入全局变量集合中
        self._variable = state_ops.variable_op_v2(shape,self._initial_value.dtype.base_dtype,name=name)
        self._initializer_op = state_ops.assign(self._variable,self._build_initializer_expr(self._initial_value),...)  
    def read_value(self):
        return array_ops.identity(self._variable, name="read")
    def eval(self, session=None):
        return self._variable.eval(session=session)
    def assign(self, value, use_locking=False):
        return state_ops.assign(self._variable, value, use_locking=use_locking)
    def assign_add(self, delta, use_locking=False):
        return state_ops.assign_add(self._variable, delta, use_locking=use_locking)
    ...
class PartitionedVariable(object):
    ...

# tensorflow/tensorflow/python/ops/variable_scope.py
class _VariableStore(object):
    def __init__(self):
        """Create a variable store."""
        self._vars = {}  # A dictionary of the stored TensorFlow variables.
        self._partitioned_vars = {}  # A dict of the stored PartitionedVariables.
        self.variable_scopes_count = {}  # Count re-used variable scopes.
    def get_variable(self, name, shape=None, dtype=dtypes.float32,...):
        def _true_getter(name, shape=None, dtype=dtypes.float32,...):
            if partitioner is not None and not is_scalar:
                return self._get_partitioned_variable(name=name,shape=shape,dtype=dtype,...)
            return self._get_single_variable(name=name, shape=shape, dtype=dtype,...)
        if custom_getter is not None:     
            return custom_getter(**custom_getter_kwargs)   
        else:                            
            return _true_getter(name, shape=shape, dtype=dtype,...)      
    def _get_single_variable(self,name,shape=None,...):
        if name in self._vars: 
            found_var = self._vars[name]
            return found_var
        if use_resource:
            v = resource_variable_ops.ResourceVariable(initial_value=init_val,name=name,...)
        else:
            v = variables.Variable(initial_value=init_val,name=name,...)
        self._vars[name] = v
        return v

# tensorflow/tensorflow/python/layers/base.py
class Layer(object):
    def __init__(self, trainable=True, name=None, dtype=None,activity_regularizer=None, **kwargs):
        self.trainable = trainable             # Whether the layer should be trained (boolean).
        self.built = False
        self.input_spec = None                  #  specifying the constraints on inputs that can be accepted by the layer.
        self._trainable_weights = []            # List of trainable variables.
        self._non_trainable_weights = []        # List of non-trainable variables.
        self._updates = []                      # List of update ops of this layer.
        self._losses = []                       # List of losses added by this layer.
        self._reuse = kwargs.get('_reuse')
        self._graph = ops.get_default_graph()
        self._dtype = ...           # Default dtype of the layer (default of `None` means use the type of the first input).
        self._name = ...            # The name of the layer (string).
        self._scope = ...
    def add_variable(self, name, shape, dtype=None,...):
        ...
        variable = vs.get_variable(name,shape=shape,...)
        if variable in existing_variables:
            return variable
        if trainable:
            self._trainable_weights.append(variable)
        else:
            self._non_trainable_weights.append(variable)
        return variable
    def __call__(self, inputs, *args, **kwargs):
        ...
        if not self.built:
            self.build(input_shapes)
        ...
        outputs = self.call(inputs, *args, **kwargs)
        ...
        # Add an inbound node to the layer, so it can keep track of this call.
        self._add_inbound_node(input_tensors=inputs, output_tensors=outputs, arguments=user_kwargs)
        ...
        self.built = True
        return outputs  
    # 一般由子类覆盖
    def call(self, inputs, **kwargs):  
        return inputs    
    def apply(self, inputs, *args, **kwargs): 
        return self.__call__(inputs, *args, **kwargs)          

# tensorflow/tensorflow/python/layers/core.py
class Dense(base.Layer):
    def build(self, input_shape):
        input_shape = tensor_shape.TensorShape(input_shape)
        self.input_spec = base.InputSpec(min_ndim=2, axes={-1: input_shape[-1].value})
        self.kernel = self.add_variable('kernel',shape=[input_shape[-1].value, self.units],...)
        if self.use_bias:
            self.bias = self.add_variable('bias',shape=[self.units,],...)
        else:
            self.bias = None
        self.built = True
  def call(self, inputs):
        inputs = ops.convert_to_tensor(inputs, dtype=self.dtype)
        ...
        outputs = standard_ops.matmul(inputs, self.kernel)
        if self.use_bias:
            outputs = nn.bias_add(outputs, self.bias)
        if self.activation is not None:
            return self.activation(outputs)  # pylint: disable=not-callable
        return outputs
def dense(inputs, units,activation=None,...):
    layer = Dense(units,activation=activation,use_bias=use_bias,...)
    return layer.apply(inputs)

# tensorflow/tensorflow/python/training/optimizer.py
class Optimizer(object):
    def minimize(self, loss, global_step=None, var_list=None,...):
        grads_and_vars = self.compute_gradients(loss, var_list=var_list, ...)
        vars_with_grad = [v for g, v in grads_and_vars if g is not None]
        return self.apply_gradients(grads_and_vars, global_step=global_step,name=name)
    # Compute gradients of `loss` for the variables in `var_list
    def compute_gradients(self, loss, var_list=None,...):
        ...
        # 根据原本计算图中所有的 op创建一个顺序的list，然后反向遍历这个list，对每个需要求导并且能够求导的op（即已经定义好了对应的梯度函数的op）调用其梯度函数，然后沿着原本计算图的方向反向串起另一部分的计算图（输入输出互换，原本的数据 Tensor 换成梯度 Tensor）
        grads = gradients.gradients(loss, var_refs, grad_ys=grad_loss,...)
        grads_and_vars = list(zip(grads, var_list))
        return grads_and_vars
    # apply_gradients函数根据前面求得的梯度，把梯度更新到参数上
    def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        converted_grads_and_vars = []
        for g, v in grads_and_vars:
            if g is not None:
                # Convert the grad to Tensor or IndexedSlices if necessary.
                g = ops.convert_to_tensor_or_indexed_slices(g)
            # _get_processor函数可理解为一种快速更新variables的方法，每个processor都会包含一个update_op这样的函数来进行variable更新操作
            p = _get_processor(v)
            converted_grads_and_vars.append((g, v, p))
        converted_grads_and_vars = tuple(converted_grads_and_vars)
        var_list = [v for g, v, _ in converted_grads_and_vars if g is not None]
        # 创建一些优化器自带的一些参数，比如AdamOptimizer的m和v
        self._create_slots([_get_variable_for(v) for v in var_list])
        update_ops = []
        self._prepare()
        for grad, var, processor in converted_grads_and_vars:
            update_ops.append(processor.update_op(self, grad))  # 核心部分
        if global_step is None:
            apply_updates = self._finish(update_ops, name)
        else:
            with ops.control_dependencies([self._finish(update_ops, "update")]): # 用来控制计算流图的，给图中的某些节点指定计算的顺序
                with ops.colocate_with(global_step):  # 保证每个参数var的更新都在同一个device上
                    apply_updates = state_ops.assign_add(global_step, 1, name=name).op
        train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
        if apply_updates not in train_op:
            train_op.append(apply_updates)
        return apply_updates
    def _resource_apply_sparse_duplicate_indices(self, grad, handle, indices):
        # _deduplicate_indexed_slices 处理重复的index， 如果当前batch的行号是[0, 201, 201, 301]，有重复的index号怎么办呢？其实只需将重复位置的梯度加起来即可(_deduplicate_indexed_slices)。
        summed_grad, unique_indices = _deduplicate_indexed_slices(values=grad, indices=indices)
        return self._resource_apply_sparse(summed_grad, handle, unique_indices)
    def _create_slots(self, var_list):
        pass

def compute_gradients(loss, grad=I):
  vrg = build_virtual_reversed_graph(loss)
  for op in vrg.topological_sort():
    # 对每个正 向子图中的 OP 寻找其「梯度函数」
    grad_fn = ops.get_gradient_function(op)
    # 调用该梯度函数，该梯度函数将构造该 OP 对 应的反向的局部子图。
    grad = grad_fn(op, grad)
def apply_gradients(grads_and_vars, learning_rate):
    for (grad, var) in grads_and_vars:
      # 对于每个 (grad_vi, vi)，构造一个更 新 vi 的子图
      apply_gradient_descent(learning_rate, grad, var)

@ops.RegisterGradient("op_name")
def op_grad_func(op, grad)

@ops.RegisterGradient("Square")
def SquareGrad(op, grad):
    x = op.inputs[0]
    with ops.control_dependencies([grad.op]):
    x = math_ops.conj(x)
    return grad * (2.0 * x)

class Optimizer(object):
  def minimize(self, loss, var_list=None, global_step=None):
    """Add operations to minimize loss by updating var_list.
    """
    grads_and_vars = self.compute_gradients(loss, var_list=var_list)
    # 将梯度apply到变量上，具体梯度如何更新到变量，由 _apply_dense、_resource_apply_dense、_apply_sparse、_resource_apply_spars这四个方法实现。
    return self.apply_gradients(grads_and_vars,global_step=global_step)

# tensorflow/tensorflow/python/training/optimizer.py
def _get_processor(v):
    """The processor of v."""
    if context.in_eager_mode():
        return _DenseResourceVariableProcessor(v)
    if v.op.type == "VarHandleOp":
        return _DenseResourceVariableProcessor(v)
    if isinstance(v, variables.Variable):
        return _RefVariableProcessor(v)
    if v.op.type == "SubmodelPort":
        return _StreamingModelPortProcessor(v)
    raise NotImplementedError("Trying to optimize unsupported type ", v)
class _DenseResourceVariableProcessor(_OptimizableVariable):
    """Processor for dense ResourceVariables."""
    def __init__(self, v):
        self._v = v
    def target(self):
        return self._v
    def update_op(self, optimizer, g):
        if isinstance(g, ops.IndexedSlices):
            return optimizer._resource_apply_sparse_duplicate_indices(g.values, self._v, g.indices) # ==> _resource_apply_sparse
        update_op = optimizer._resource_apply_dense(g, self._v)
        return update_op

# tensorflow/tensorflow/python/framework/ops.py
# This class is a simple wrapper for a pair of `Tensor` objects:
class IndexedSlices(_TensorLike):
    def __init__(self, values, indices, dense_shape=None):
    """Creates an `IndexedSlices`."""
    _get_graph_from_inputs([values, indices, dense_shape])
    self._indices = indices         # 前向传播中取的那几个位置，也就是最后要更新的那几个位置
    self._values = values           # 这些位置所对应的梯度值
    self._dense_shape = dense_shape # 矩阵原本的形状
# tensorflow/tensorflow/python/framework/sparse_tensor.py
class SparseTensor(_TensorLike):
    def __init__(self, indices, values, dense_shape):
      ...
      self._indices = indices
      self._values = values
      self._dense_shape = dense_shape

class GradientDescentOptimizer(optimizer.Optimizer):
    def __init__(self, learning_rate, use_locking=False, name="GradientDescent"):
        super(GradientDescentOptimizer, self).__init__(use_locking, name)
        self._learning_rate = learning_rate
  def _apply_dense(self, grad, var):
        return training_ops.apply_gradient_descent(
            var,
            math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype),
            grad,
            use_locking=self._use_locking).op
  def _resource_apply_dense(self, grad, handle):
        return training_ops.resource_apply_gradient_descent(
            handle.handle, math_ops.cast(self._learning_rate_tensor,
                                        grad.dtype.base_dtype),
            grad, use_locking=self._use_locking)
  def _resource_apply_sparse_duplicate_indices(self, grad, handle, indices):
        return resource_variable_ops.resource_scatter_add(handle.handle, indices, -grad * self._learning_rate)

  def _apply_sparse_duplicate_indices(self, grad, var):
        delta = ops.IndexedSlices(
            grad.values *
            math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype),
            grad.indices, grad.dense_shape)
        return var.scatter_sub(delta, use_locking=self._use_locking)
  def _prepare(self):
    self._learning_rate_tensor = ops.convert_to_tensor(self._learning_rate,name="learning_rate")

# tensorflow/tensorflow/python/framework/ops.py
class Graph(object):
    def __init__(self):
        self._lock = threading.Lock()   # 加线程锁，使得注册op时，不会有其他线程注册op到graph中，从而保证共享graph是线程安全的
        # op相关数据。
        # 为graph的每个op分配一个id，通过id可以快速索引到相关op。故创建了_nodes_by_id字典
        self._nodes_by_id = dict()      # GUARDED_BY(self._lock)
        self._next_id_counter = 0       # GUARDED_BY(self._lock)
        # 同时也可以通过name来快速索引op，故创建了_nodes_by_name字典
        self._nodes_by_name = dict()    # GUARDED_BY(self._lock)
        self._version = 0               # GUARDED_BY(self._lock)
        
        # tensor相关数据。
        self._handle_feeders = {}   # 处理tensor的placeholder
        self._handle_readers = {}   # 处理tensor的read操作
        self._handle_movers = {}    # 处理tensor的move操作
        self._handle_deleters = {}  # 处理tensor的delete操作
    # graph添加op
    def _add_op(self, op):
        self._check_not_finalized()     # graph被设置为final后，就是只读的了，不能添加op了。
        with self._lock:                # 保证共享graph的线程安全
            # 将op以id和name分别构建字典，添加到_nodes_by_id和_nodes_by_name字典中，方便后续快速索引
            self._nodes_by_id[op._id] = op
            self._nodes_by_name[op.name] = op
            self._version = max(self._version, op._id)

class Operation(object):
  def __init__(self,node_def,g,inputs=None,...):
    self._graph = g        # graph引用，通过它可以拿到Operation所注册到的Graph
    
    # inputs
    if inputs is None:
      inputs = []
    #  input types
    if input_types is None:
      input_types = [i.dtype.base_dtype for i in inputs]

    control_input_ops = []
    
    # node_def和op_def是两个最关键的成员
    if not self._graph._c_graph:
      self._inputs_val = list(inputs)  # Defensive copy.
      self._input_types_val = input_types
      self._control_inputs_val = control_input_ops

      # NodeDef，深复制
      self._node_def_val = copy.deepcopy(node_def)
      # OpDef
      self._op_def_val = op_def
    # outputs输出
    self._outputs = [
        Tensor(self, i, output_type)
        for i, output_type in enumerate(output_types)
    ]
  def name(self):
    return self._node_def_val.name
  def _id(self):
    return self._id_value
  def device(self):
    return self._node_def_val.device
  def node_def(self):   # NodeDef成员，获取Operation的动态属性信息，例如Operation分配到的设备信息，Operation的name等
    return self._node_def_val
  def op_def(self):     # OpDef，获取Operation的静态属性信息，例如Operation入参列表，出参列表等
    return self._op_def_val

class Tensor(_TensorLike):
  def __init__(self, op, value_index, dtype):
    self._op = op       # 源节点，tensor的生产者，会计算得到tensor
    # tensor在源节点的输出边集合中的索引。源节点可能会有多条输出边
    self._value_index = value_index         # 利用op和value_index即可唯一确定tensor。
    self._dtype = dtypes.as_dtype(dtype)    # tensor中保存的数据的数据类型
    self._shape_val = tensor_shape.unknown_shape()  # tensor的shape，可以得到张量的rank，维度等信息
    self._consumers = []                            # 目标节点列表，tensor的消费者，会使用该tensor来进行计算
    self._handle_data = None
    self._id = uid()

# 在Python脚本中定义matmul运算
tf.manual(a,b)
    # 根据Ops名称MatMul从Ops库中找出对应Ops类型
    _op_def_lib.apply_op('MatMul',a=a,b=b,...)
    # ops/op_def_library.py
    # 创建ops节点
    graph.create_op(op_type_name,inputs,output_types,...)
    # framework/ops.py
    # 创建ops节点并指定相关属性和设备分配
    node_def = _NodeDef(op_type,name,device=None,attrs=attrs)
    ret = Operation(node_def,self,inputs=inputs,...)