From 4d9809c1765e972645017b03bd52c16b1e0bcdf4 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 11 May 2020 08:21:04 +0200 Subject: [PATCH 001/227] Added test for droput_sparse --- .../decagon_pytorch}/__init__.py | 0 .../decagon_pytorch}/convolve.py | 0 .../decagon_pytorch}/dropout.py | 0 .../decagon_pytorch}/model.py | 0 .../decagon_pytorch}/weights.py | 0 tests/decagon_pytorch/test_dropout.py | 34 +++++++++++++++++++ 6 files changed, 34 insertions(+) rename {decagon_pytorch => src/decagon_pytorch}/__init__.py (100%) rename {decagon_pytorch => src/decagon_pytorch}/convolve.py (100%) mode change 100755 => 100644 rename {decagon_pytorch => src/decagon_pytorch}/dropout.py (100%) rename {decagon_pytorch => src/decagon_pytorch}/model.py (100%) rename {decagon_pytorch => src/decagon_pytorch}/weights.py (100%) create mode 100644 tests/decagon_pytorch/test_dropout.py diff --git a/decagon_pytorch/__init__.py b/src/decagon_pytorch/__init__.py similarity index 100% rename from decagon_pytorch/__init__.py rename to src/decagon_pytorch/__init__.py diff --git a/decagon_pytorch/convolve.py b/src/decagon_pytorch/convolve.py old mode 100755 new mode 100644 similarity index 100% rename from decagon_pytorch/convolve.py rename to src/decagon_pytorch/convolve.py diff --git a/decagon_pytorch/dropout.py b/src/decagon_pytorch/dropout.py similarity index 100% rename from decagon_pytorch/dropout.py rename to src/decagon_pytorch/dropout.py diff --git a/decagon_pytorch/model.py b/src/decagon_pytorch/model.py similarity index 100% rename from decagon_pytorch/model.py rename to src/decagon_pytorch/model.py diff --git a/decagon_pytorch/weights.py b/src/decagon_pytorch/weights.py similarity index 100% rename from decagon_pytorch/weights.py rename to src/decagon_pytorch/weights.py diff --git a/tests/decagon_pytorch/test_dropout.py b/tests/decagon_pytorch/test_dropout.py new file mode 100644 index 0000000..60366ec --- /dev/null +++ b/tests/decagon_pytorch/test_dropout.py @@ -0,0 +1,34 @@ +from decagon_pytorch.dropout import dropout_sparse +import torch +import numpy as np + + +def dropout_dense(a, keep_prob): + i = np.array(np.where(a)) + v = a[i[0, :], i[1, :]] + + # torch.random.manual_seed(0) + n = keep_prob + torch.rand(len(v)) + n = torch.floor(n).to(torch.bool) + i = i[:, n] + v = v[n] + x = torch.sparse_coo_tensor(i, v, size=a.shape) + + return x * (1./keep_prob) + + +def test_dropout_sparse(): + for i in range(11): + torch.random.manual_seed(i) + a = torch.rand((5, 10)) + a[a < .5] = 0 + + keep_prob=i/10. + np.finfo(np.float32).eps + + torch.random.manual_seed(i) + b = dropout_dense(a, keep_prob=keep_prob) + + torch.random.manual_seed(i) + c = dropout_sparse(a.to_sparse(), keep_prob=keep_prob) + + assert np.all(np.array(b.to_dense()) == np.array(c.to_dense())) -- 2.26.2 From d3de9b913d847abc182dd7208b44b72ddeb46935 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 11 May 2020 12:50:35 +0200 Subject: [PATCH 002/227] Started implementing convolutions, with tests. --- .gitignore | 2 +- src/decagon_pytorch/convolve.py | 53 ++++++++++++++++++++ src/decagon_pytorch/weights.py | 6 +-- tests/decagon_pytorch/test_convolve.py | 69 ++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 tests/decagon_pytorch/test_convolve.py diff --git a/.gitignore b/.gitignore index c20c2ab..837e16b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ __pycache__ - +.cache/ diff --git a/src/decagon_pytorch/convolve.py b/src/decagon_pytorch/convolve.py index e69de29..d125e98 100644 --- a/src/decagon_pytorch/convolve.py +++ b/src/decagon_pytorch/convolve.py @@ -0,0 +1,53 @@ +import torch +from .dropout import dropout_sparse +from .weights import init_glorot + + +class SparseGraphConv(torch.nn.Module): + """Convolution layer for sparse inputs.""" + def __init__(self, in_channels, out_channels, + adjacency_matrix, **kwargs): + super().__init__(**kwargs) + self.in_channels = in_channels + self.out_channels = out_channels + self.weight = init_glorot(in_channels, out_channels) + self.adjacency_matrix = adjacency_matrix + + + def forward(self, x): + x = torch.sparse.mm(x, self.weight) + x = torch.sparse.mm(self.adjacency_matrix, x) + return x + + +class SparseDropoutGraphConvActivation(torch.nn.Module): + def __init__(self, input_dim, output_dim, + adjacency_matrix, keep_prob=1., + activation=torch.nn.functional.relu, + **kwargs): + super().__init__(**kwargs) + self.sparse_graph_conv = SparseGraphConv(input_dim, output_dim) + self.keep_prob = keep_prob + self.activation = activation + + def forward(self, x): + x = dropout_sparse(x, self.keep_prob) + x = self.sparse_graph_conv(x) + x = self.activation(x) + return x + + +class SparseMultiDGCA(torch.nn.Module): + def __init__(self, input_dim, output_dim, + adjacency_matrices, keep_prob=1., + activation=torch.nn.functional.relu, + **kwargs): + super().__init__(**kwargs) + self.sparse_dgca = [ SparseDropoutGraphConvActivation(input_dim, output_dim, adj_mat, keep_prob, activation) for adj_mat in adjacency_matrices ] + + def forward(self, x): + out = torch.zeros(len(x), output_dim, dtype=x.dtype) + for f in self.sparse_dgca: + out += f(x) + out = torch.nn.functional.normalize(out, p=2, dim=1) + return out diff --git a/src/decagon_pytorch/weights.py b/src/decagon_pytorch/weights.py index 305a70f..cfab885 100644 --- a/src/decagon_pytorch/weights.py +++ b/src/decagon_pytorch/weights.py @@ -2,12 +2,12 @@ import torch import numpy as np -def init_glorot(input_dim, output_dim): +def init_glorot(in_channels, out_channels, dtype=torch.float32): """Create a weight variable with Glorot & Bengio (AISTATS 2010) initialization. """ - init_range = np.sqrt(6.0 / (input_dim + output_dim)) + init_range = np.sqrt(6.0 / (in_channels + out_channels)) initial = -init_range + 2 * init_range * \ - torch.rand(( input_dim, output_dim ), dtype=torch.float32) + torch.rand(( in_channels, out_channels ), dtype=dtype) initial = initial.requires_grad_(True) return initial diff --git a/tests/decagon_pytorch/test_convolve.py b/tests/decagon_pytorch/test_convolve.py new file mode 100644 index 0000000..611f0a1 --- /dev/null +++ b/tests/decagon_pytorch/test_convolve.py @@ -0,0 +1,69 @@ +import decagon_pytorch.convolve +import decagon.deep.layers +import torch +import tensorflow as tf +import numpy as np + + +def prepare_data(): + np.random.seed(0) + latent = np.random.random((5, 10)).astype(np.float32) + latent[latent < .5] = 0 + latent = np.ceil(latent) + adjacency_matrices = [] + for _ in range(5): + adj_mat = np.random.random((len(latent),) * 2).astype(np.float32) + adj_mat[adj_mat < .5] = 0 + adj_mat = np.ceil(adj_mat) + adjacency_matrices.append(adj_mat) + return latent, adjacency_matrices + + +def sparse_graph_conv_torch(): + torch.random.manual_seed(0) + latent, adjacency_matrices = prepare_data() + print('latent.dtype:', latent.dtype) + latent = torch.tensor(latent).to_sparse() + adj_mat = adjacency_matrices[0] + adj_mat = torch.tensor(adj_mat).to_sparse() + print('adj_mat.dtype:', adj_mat.dtype, + 'latent.dtype:', latent.dtype) + conv = decagon_pytorch.convolve.SparseGraphConv(10, 10, + adj_mat) + latent = conv(latent) + return latent + + +def dense_to_sparse_tf(x): + a, b = np.where(x) + indices = np.array([a, b]).T + values = x[a, b] + return tf.sparse.SparseTensor(indices, values, x.shape) + + + +def sparse_graph_conv_tf(): + torch.random.manual_seed(0) + latent, adjacency_matrices = prepare_data() + conv_torch = decagon_pytorch.convolve.SparseGraphConv(10, 10, + torch.tensor(adjacency_matrices[0]).to_sparse()) + weight = tf.constant(conv_torch.weight.detach().numpy()) + latent = dense_to_sparse_tf(latent) + adj_mat = dense_to_sparse_tf(adjacency_matrices[0]) + latent = tf.sparse_tensor_dense_matmul(latent, weight) + latent = tf.sparse_tensor_dense_matmul(adj_mat, latent) + return latent + + +def test_sparse_graph_conv(): + latent_torch = sparse_graph_conv_torch() + latent_tf = sparse_graph_conv_tf() + assert np.all(latent_torch.detach().numpy() == latent_tf.eval(session = tf.Session())) + + +def test_sparse_dropout_grap_conv_activation(): + pass + + +def test_sparse_multi_dgca(): + pass -- 2.26.2 From 2d92cc95bbb3cd3e643863cd8d9371d472da44a4 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 11 May 2020 14:38:55 +0200 Subject: [PATCH 003/227] Add test_sparse_dropout_grap_conv_activation() --- src/decagon_pytorch/convolve.py | 2 +- tests/decagon_pytorch/test_convolve.py | 72 +++++++++++++++++++++++--- 2 files changed, 66 insertions(+), 8 deletions(-) diff --git a/src/decagon_pytorch/convolve.py b/src/decagon_pytorch/convolve.py index d125e98..aec9c04 100644 --- a/src/decagon_pytorch/convolve.py +++ b/src/decagon_pytorch/convolve.py @@ -26,7 +26,7 @@ class SparseDropoutGraphConvActivation(torch.nn.Module): activation=torch.nn.functional.relu, **kwargs): super().__init__(**kwargs) - self.sparse_graph_conv = SparseGraphConv(input_dim, output_dim) + self.sparse_graph_conv = SparseGraphConv(input_dim, output_dim, adjacency_matrix) self.keep_prob = keep_prob self.activation = activation diff --git a/tests/decagon_pytorch/test_convolve.py b/tests/decagon_pytorch/test_convolve.py index 611f0a1..2bf603a 100644 --- a/tests/decagon_pytorch/test_convolve.py +++ b/tests/decagon_pytorch/test_convolve.py @@ -19,6 +19,26 @@ def prepare_data(): return latent, adjacency_matrices +def dense_to_sparse_tf(x): + a, b = np.where(x) + indices = np.array([a, b]).T + values = x[a, b] + return tf.sparse.SparseTensor(indices, values, x.shape) + + +def dropout_sparse_tf(x, keep_prob, num_nonzero_elems): + """Dropout for sparse tensors. Currently fails for very large sparse tensors (>1M elements) + """ + noise_shape = [num_nonzero_elems] + random_tensor = keep_prob + random_tensor += tf.convert_to_tensor(torch.rand(noise_shape).detach().numpy()) + # tf.convert_to_tensor(np.random.random(noise_shape)) + # tf.random_uniform(noise_shape) + dropout_mask = tf.cast(tf.floor(random_tensor), dtype=tf.bool) + pre_out = tf.sparse_retain(x, dropout_mask) + return pre_out * (1./keep_prob) + + def sparse_graph_conv_torch(): torch.random.manual_seed(0) latent, adjacency_matrices = prepare_data() @@ -34,24 +54,51 @@ def sparse_graph_conv_torch(): return latent -def dense_to_sparse_tf(x): - a, b = np.where(x) - indices = np.array([a, b]).T - values = x[a, b] - return tf.sparse.SparseTensor(indices, values, x.shape) +def sparse_graph_conv_tf(): + torch.random.manual_seed(0) + latent, adjacency_matrices = prepare_data() + conv_torch = decagon_pytorch.convolve.SparseGraphConv(10, 10, + torch.tensor(adjacency_matrices[0]).to_sparse()) + weight = tf.constant(conv_torch.weight.detach().numpy()) + latent = dense_to_sparse_tf(latent) + adj_mat = dense_to_sparse_tf(adjacency_matrices[0]) + latent = tf.sparse_tensor_dense_matmul(latent, weight) + latent = tf.sparse_tensor_dense_matmul(adj_mat, latent) + return latent +def sparse_dropout_graph_conv_activation_torch(keep_prob=1.): + torch.random.manual_seed(0) + latent, adjacency_matrices = prepare_data() + latent = torch.tensor(latent).to_sparse() + adj_mat = adjacency_matrices[0] + adj_mat = torch.tensor(adj_mat).to_sparse() + conv = decagon_pytorch.convolve.SparseDropoutGraphConvActivation(10, 10, + adj_mat, keep_prob=keep_prob) + latent = conv(latent) + return latent -def sparse_graph_conv_tf(): + +def sparse_dropout_graph_conv_activation_tf(keep_prob=1.): torch.random.manual_seed(0) latent, adjacency_matrices = prepare_data() conv_torch = decagon_pytorch.convolve.SparseGraphConv(10, 10, torch.tensor(adjacency_matrices[0]).to_sparse()) + weight = tf.constant(conv_torch.weight.detach().numpy()) + nonzero_feat = np.sum(latent > 0) + latent = dense_to_sparse_tf(latent) + latent = dropout_sparse_tf(latent, keep_prob, + nonzero_feat) + adj_mat = dense_to_sparse_tf(adjacency_matrices[0]) + latent = tf.sparse_tensor_dense_matmul(latent, weight) latent = tf.sparse_tensor_dense_matmul(adj_mat, latent) + + latent = tf.nn.relu(latent) + return latent @@ -62,7 +109,18 @@ def test_sparse_graph_conv(): def test_sparse_dropout_grap_conv_activation(): - pass + for i in range(11): + keep_prob = i/10. + np.finfo(np.float32).eps + + latent_torch = sparse_dropout_graph_conv_activation_torch(keep_prob) + latent_tf = sparse_dropout_graph_conv_activation_tf(keep_prob) + + latent_torch = latent_torch.detach().numpy() + latent_tf = latent_tf.eval(session = tf.Session()) + print('latent_torch:', latent_torch) + print('latent_tf:', latent_tf) + + assert np.all(latent_torch - latent_tf < .000001) def test_sparse_multi_dgca(): -- 2.26.2 From 7f085b372a84bfc4aaa8efa8c04260b41d9a4820 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 11 May 2020 18:13:59 +0200 Subject: [PATCH 004/227] Add test_sparse_multi_dgca() --- tests/decagon_pytorch/test_convolve.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/decagon_pytorch/test_convolve.py b/tests/decagon_pytorch/test_convolve.py index 2bf603a..1624a80 100644 --- a/tests/decagon_pytorch/test_convolve.py +++ b/tests/decagon_pytorch/test_convolve.py @@ -124,4 +124,23 @@ def test_sparse_dropout_grap_conv_activation(): def test_sparse_multi_dgca(): - pass + latent_torch = None + latent_tf = [] + + for i in range(11): + keep_prob = i/10. + np.finfo(np.float32).eps + + latent_torch = sparse_dropout_graph_conv_activation_torch(keep_prob) \ + if latent_torch is None \ + else latent_torch + sparse_dropout_graph_conv_activation_torch(keep_prob) + + latent_tf.append(sparse_dropout_graph_conv_activation_tf(keep_prob)) + + latent_torch = torch.nn.functional.normalize(latent_torch, p=2, dim=1) + latent_tf = tf.add_n(latent_tf) + latent_tf = tf.nn.l2_normalize(latent_tf, dim=1) + + latent_torch = latent_torch.detach().numpy() + latent_tf = latent_tf.eval(session = tf.Session()) + + assert np.all(latent_torch - latent_tf < .000001) -- 2.26.2 From 01134b893c20298347869a600ec5559a9a014180 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 12 May 2020 18:06:07 +0200 Subject: [PATCH 005/227] Started implementing dense convolutions. --- src/decagon_pytorch/convolve.py | 50 +++++++++++++++++++++++++- tests/decagon_pytorch/test_convolve.py | 19 ++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/src/decagon_pytorch/convolve.py b/src/decagon_pytorch/convolve.py index aec9c04..44252f3 100644 --- a/src/decagon_pytorch/convolve.py +++ b/src/decagon_pytorch/convolve.py @@ -46,8 +46,56 @@ class SparseMultiDGCA(torch.nn.Module): self.sparse_dgca = [ SparseDropoutGraphConvActivation(input_dim, output_dim, adj_mat, keep_prob, activation) for adj_mat in adjacency_matrices ] def forward(self, x): - out = torch.zeros(len(x), output_dim, dtype=x.dtype) + out = torch.zeros(len(x), self.output_dim, dtype=x.dtype) for f in self.sparse_dgca: out += f(x) out = torch.nn.functional.normalize(out, p=2, dim=1) return out + + +class GraphConv(torch.nn.Module): + def __init__(self, in_channels, out_channels, + adjacency_matrix, **kwargs): + super().__init__(**kwargs) + self.in_channels = in_channels + self.out_channels = out_channels + self.weight = init_glorot(in_channels, out_channels) + self.adjacency_matrix = adjacency_matrix + + def forward(self, x): + x = torch.mm(x, self.weight) + x = torch.mm(self.adjacency_matrix, x) + return x + + +class DropoutGraphConvActivation(torch.nn.Module): + def __init__(self, input_dim, output_dim, + adjacency_matrix, keep_prob=1., + activation=torch.nn.functional.relu, + **kwargs): + super().__init__(**kwargs) + self.graph_conv = GraphConv(input_dim, output_dim, adjacency_matrix) + self.keep_prob = keep_prob + self.activation = activation + + def forward(self, x): + x = torch.nn.functional.dropout(x, 1.-self.keep_prob) + x = self.graph_conv(x) + x = self.activation(x) + return x + + +class MultiDGCA(torch.nn.Module): + def __init__(self, input_dim, output_dim, + adjacency_matrices, keep_prob=1., + activation=torch.nn.functional.relu, + **kwargs): + super().__init__(**kwargs) + self.dgca = [ DropoutGraphConvActivation(input_dim, output_dim, adj_mat, keep_prob, activation) for adj_mat in adjacency_matrices ] + + def forward(self, x): + out = torch.zeros(len(x), self.output_dim, dtype=x.dtype) + for f in self.dgca: + out += f(x) + out = torch.nn.functional.normalize(out, p=2, dim=1) + return out diff --git a/tests/decagon_pytorch/test_convolve.py b/tests/decagon_pytorch/test_convolve.py index 1624a80..3adb9fd 100644 --- a/tests/decagon_pytorch/test_convolve.py +++ b/tests/decagon_pytorch/test_convolve.py @@ -39,6 +39,18 @@ def dropout_sparse_tf(x, keep_prob, num_nonzero_elems): return pre_out * (1./keep_prob) +def graph_conv_torch(): + torch.random.manual_seed(0) + latent, adjacency_matrices = prepare_data() + latent = torch.tensor(latent) + adj_mat = adjacency_matrices[0] + adj_mat = torch.tensor(adj_mat) + conv = decagon_pytorch.convolve.GraphConv(10, 10, + adj_mat) + latent = conv(latent) + return latent + + def sparse_graph_conv_torch(): torch.random.manual_seed(0) latent, adjacency_matrices = prepare_data() @@ -144,3 +156,10 @@ def test_sparse_multi_dgca(): latent_tf = latent_tf.eval(session = tf.Session()) assert np.all(latent_torch - latent_tf < .000001) + + +def test_graph_conv(): + latent_dense = graph_conv_torch() + latent_sparse = sparse_graph_conv_torch() + + assert np.all(latent_dense.detach().numpy() == latent_sparse.detach().numpy()) -- 2.26.2 From 8f41021d69df22f2d4fe902db9fc5f8661102a00 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 12 May 2020 19:47:07 +0200 Subject: [PATCH 006/227] Add test_dropout_graph_conv_activation(). --- src/decagon_pytorch/convolve.py | 5 ++- src/decagon_pytorch/dropout.py | 13 +++++++ tests/decagon_pytorch/test_convolve.py | 52 +++++++++++++++++++++++++- 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/decagon_pytorch/convolve.py b/src/decagon_pytorch/convolve.py index 44252f3..c781e6b 100644 --- a/src/decagon_pytorch/convolve.py +++ b/src/decagon_pytorch/convolve.py @@ -1,5 +1,6 @@ import torch -from .dropout import dropout_sparse +from .dropout import dropout_sparse, \ + dropout from .weights import init_glorot @@ -79,7 +80,7 @@ class DropoutGraphConvActivation(torch.nn.Module): self.activation = activation def forward(self, x): - x = torch.nn.functional.dropout(x, 1.-self.keep_prob) + x = dropout(x, keep_prob=self.keep_prob) x = self.graph_conv(x) x = self.activation(x) return x diff --git a/src/decagon_pytorch/dropout.py b/src/decagon_pytorch/dropout.py index 3162572..27196fe 100644 --- a/src/decagon_pytorch/dropout.py +++ b/src/decagon_pytorch/dropout.py @@ -16,3 +16,16 @@ def dropout_sparse(x, keep_prob): x = torch.sparse_coo_tensor(i, v, size=size) return x * (1./keep_prob) + + +def dropout(x, keep_prob): + """Dropout for dense tensors. + """ + shape = x.shape + x = torch.flatten(x) + n = keep_prob + torch.rand(len(x)) + n = (1. - torch.floor(n)).to(torch.bool) + x[n] = 0 + x = torch.reshape(x, shape) + # x = torch.nn.functional.dropout(x, p=1.-keep_prob) + return x * (1./keep_prob) diff --git a/tests/decagon_pytorch/test_convolve.py b/tests/decagon_pytorch/test_convolve.py index 3adb9fd..74998d0 100644 --- a/tests/decagon_pytorch/test_convolve.py +++ b/tests/decagon_pytorch/test_convolve.py @@ -16,6 +16,8 @@ def prepare_data(): adj_mat[adj_mat < .5] = 0 adj_mat = np.ceil(adj_mat) adjacency_matrices.append(adj_mat) + print('latent:', latent) + print('adjacency_matrices[0]:', adjacency_matrices[0]) return latent, adjacency_matrices @@ -51,6 +53,18 @@ def graph_conv_torch(): return latent +def dropout_graph_conv_activation_torch(keep_prob=1.): + torch.random.manual_seed(0) + latent, adjacency_matrices = prepare_data() + latent = torch.tensor(latent) + adj_mat = adjacency_matrices[0] + adj_mat = torch.tensor(adj_mat) + conv = decagon_pytorch.convolve.DropoutGraphConvActivation(10, 10, + adj_mat, keep_prob=keep_prob) + latent = conv(latent) + return latent + + def sparse_graph_conv_torch(): torch.random.manual_seed(0) latent, adjacency_matrices = prepare_data() @@ -120,7 +134,7 @@ def test_sparse_graph_conv(): assert np.all(latent_torch.detach().numpy() == latent_tf.eval(session = tf.Session())) -def test_sparse_dropout_grap_conv_activation(): +def test_sparse_dropout_graph_conv_activation(): for i in range(11): keep_prob = i/10. + np.finfo(np.float32).eps @@ -163,3 +177,39 @@ def test_graph_conv(): latent_sparse = sparse_graph_conv_torch() assert np.all(latent_dense.detach().numpy() == latent_sparse.detach().numpy()) + + +def setup_function(fun): + if fun == test_dropout_graph_conv_activation: + setup_function.old_dropout = decagon_pytorch.convolve.dropout, \ + decagon_pytorch.convolve.dropout_sparse + + decagon_pytorch.convolve.dropout = lambda x, keep_prob: x + decagon_pytorch.convolve.dropout_sparse = lambda x, keep_prob: x + + +def teardown_function(fun): + if fun == test_dropout_graph_conv_activation: + decagon_pytorch.convolve.dropout, \ + decagon_pytorch.convolve.dropout_sparse = \ + setup_function.old_dropout + + +def test_dropout_graph_conv_activation(): + for i in range(11): + keep_prob = i/10. + if keep_prob == 0: + keep_prob += np.finfo(np.float32).eps + print('keep_prob:', keep_prob) + + latent_dense = dropout_graph_conv_activation_torch(keep_prob) + latent_dense = latent_dense.detach().numpy() + print('latent_dense:', latent_dense) + + latent_sparse = sparse_dropout_graph_conv_activation_torch(keep_prob) + latent_sparse = latent_sparse.detach().numpy() + print('latent_sparse:', latent_sparse) + + nonzero = (latent_dense != 0) & (latent_sparse != 0) + + assert np.all(latent_dense[nonzero] == latent_sparse[nonzero]) -- 2.26.2 From f8cec09a09d778f18fea7cb3a536acc7fb033cde Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 12 May 2020 20:06:11 +0200 Subject: [PATCH 007/227] Need to de-bug test_multi_dgca(). --- src/decagon_pytorch/convolve.py | 2 ++ tests/decagon_pytorch/test_convolve.py | 31 ++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/decagon_pytorch/convolve.py b/src/decagon_pytorch/convolve.py index c781e6b..3b116e7 100644 --- a/src/decagon_pytorch/convolve.py +++ b/src/decagon_pytorch/convolve.py @@ -44,6 +44,7 @@ class SparseMultiDGCA(torch.nn.Module): activation=torch.nn.functional.relu, **kwargs): super().__init__(**kwargs) + self.output_dim = output_dim self.sparse_dgca = [ SparseDropoutGraphConvActivation(input_dim, output_dim, adj_mat, keep_prob, activation) for adj_mat in adjacency_matrices ] def forward(self, x): @@ -92,6 +93,7 @@ class MultiDGCA(torch.nn.Module): activation=torch.nn.functional.relu, **kwargs): super().__init__(**kwargs) + self.output_dim = output_dim self.dgca = [ DropoutGraphConvActivation(input_dim, output_dim, adj_mat, keep_prob, activation) for adj_mat in adjacency_matrices ] def forward(self, x): diff --git a/tests/decagon_pytorch/test_convolve.py b/tests/decagon_pytorch/test_convolve.py index 74998d0..1e2c538 100644 --- a/tests/decagon_pytorch/test_convolve.py +++ b/tests/decagon_pytorch/test_convolve.py @@ -180,7 +180,9 @@ def test_graph_conv(): def setup_function(fun): - if fun == test_dropout_graph_conv_activation: + if fun == test_dropout_graph_conv_activation or \ + fun == test_multi_dgca: + print('Disabling dropout for testing...') setup_function.old_dropout = decagon_pytorch.convolve.dropout, \ decagon_pytorch.convolve.dropout_sparse @@ -189,7 +191,9 @@ def setup_function(fun): def teardown_function(fun): - if fun == test_dropout_graph_conv_activation: + print('Re-enabling dropout...') + if fun == test_dropout_graph_conv_activation or \ + fun == test_multi_dgca: decagon_pytorch.convolve.dropout, \ decagon_pytorch.convolve.dropout_sparse = \ setup_function.old_dropout @@ -213,3 +217,26 @@ def test_dropout_graph_conv_activation(): nonzero = (latent_dense != 0) & (latent_sparse != 0) assert np.all(latent_dense[nonzero] == latent_sparse[nonzero]) + + +def test_multi_dgca(): + keep_prob = .5 + + torch.random.manual_seed(0) + latent, adjacency_matrices = prepare_data() + + latent_sparse = torch.tensor(latent).to_sparse() + latent = torch.tensor(latent) + + adjacency_matrices_sparse = [ torch.tensor(a).to_sparse() for a in adjacency_matrices ] + adjacency_matrices = [ torch.tensor(a) for a in adjacency_matrices ] + + multi_sparse = decagon_pytorch.convolve.SparseMultiDGCA(10, 10, adjacency_matrices_sparse, keep_prob=keep_prob) + multi = decagon_pytorch.convolve.MultiDGCA(10, 10, adjacency_matrices, keep_prob=keep_prob) + + # torch.random.manual_seed(0) + latent_sparse = multi_sparse(latent_sparse) + # torch.random.manual_seed(0) + latent = multi(latent) + + assert np.all(latent_sparse.detach().numpy() - latent.detach().numpy() < .000001) -- 2.26.2 From 1d271acab8b633fa0db1e14e2e48505eb2a1bb17 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 12 May 2020 22:32:07 +0200 Subject: [PATCH 008/227] test_multi_dgca() passes. --- tests/decagon_pytorch/test_convolve.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/decagon_pytorch/test_convolve.py b/tests/decagon_pytorch/test_convolve.py index 1e2c538..210a4e5 100644 --- a/tests/decagon_pytorch/test_convolve.py +++ b/tests/decagon_pytorch/test_convolve.py @@ -227,16 +227,26 @@ def test_multi_dgca(): latent_sparse = torch.tensor(latent).to_sparse() latent = torch.tensor(latent) + assert np.all(latent_sparse.to_dense().numpy() == latent.numpy()) adjacency_matrices_sparse = [ torch.tensor(a).to_sparse() for a in adjacency_matrices ] adjacency_matrices = [ torch.tensor(a) for a in adjacency_matrices ] + for i in range(len(adjacency_matrices)): + assert np.all(adjacency_matrices[i].numpy() == adjacency_matrices_sparse[i].to_dense().numpy()) + + torch.random.manual_seed(0) multi_sparse = decagon_pytorch.convolve.SparseMultiDGCA(10, 10, adjacency_matrices_sparse, keep_prob=keep_prob) + + torch.random.manual_seed(0) multi = decagon_pytorch.convolve.MultiDGCA(10, 10, adjacency_matrices, keep_prob=keep_prob) + for i in range(len(adjacency_matrices)): + assert np.all(multi_sparse.sparse_dgca[i].sparse_graph_conv.weight.detach().numpy() == multi.dgca[i].graph_conv.weight.detach().numpy()) + # torch.random.manual_seed(0) latent_sparse = multi_sparse(latent_sparse) # torch.random.manual_seed(0) latent = multi(latent) - assert np.all(latent_sparse.detach().numpy() - latent.detach().numpy() < .000001) + assert np.all(latent_sparse.detach().numpy() == latent.detach().numpy()) -- 2.26.2 From 353fc8913e6ab3706d1298931ddb82ff2ed93fc7 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 13 May 2020 17:59:05 +0200 Subject: [PATCH 009/227] Added DEDICOMDecoder with test. --- src/decagon_pytorch/decode.py | 37 +++++++++++++++++++++++++ tests/decagon_pytorch/test_decode.py | 41 ++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 src/decagon_pytorch/decode.py create mode 100644 tests/decagon_pytorch/test_decode.py diff --git a/src/decagon_pytorch/decode.py b/src/decagon_pytorch/decode.py new file mode 100644 index 0000000..8ff6096 --- /dev/null +++ b/src/decagon_pytorch/decode.py @@ -0,0 +1,37 @@ +import torch +from .weights import init_glorot +from .dropout import dropout + + +class DEDICOMDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, drop_prob=0., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.drop_prob = drop_prob + self.activation = activation + + + self.global_interaction = init_glorot(input_dim, input_dim) + self.local_variation = [ + torch.flatten(init_glorot(input_dim, 1)) \ + for _ in range(num_relation_types) + ] + + def forward(self, inputs_row, inputs_col): + outputs = [] + for k in range(self.num_relation_types): + inputs_row = dropout(inputs_row, 1.-self.drop_prob) + inputs_col = dropout(inputs_col, 1.-self.drop_prob) + + relation = torch.diag(self.local_variation[k]) + + product1 = torch.mm(inputs_row, relation) + product2 = torch.mm(product1, self.global_interaction) + product3 = torch.mm(product2, relation) + rec = torch.mm(product3, torch.transpose(inputs_col, 0, 1)) + outputs.append(self.activation(rec)) + return outputs diff --git a/tests/decagon_pytorch/test_decode.py b/tests/decagon_pytorch/test_decode.py new file mode 100644 index 0000000..7e8c4ed --- /dev/null +++ b/tests/decagon_pytorch/test_decode.py @@ -0,0 +1,41 @@ +import decagon_pytorch.decode +import decagon.deep.layers +import numpy as np +import tensorflow as tf +import torch + + +def test_dedicom(): + dedicom_torch = decagon_pytorch.decode.DEDICOMDecoder(input_dim=10, + num_relation_types=7) + dedicom_tf = decagon.deep.layers.DEDICOMDecoder(input_dim=10, num_types=7, + edge_type=(0, 0)) + + dedicom_tf.vars['global_interaction'] = \ + tf.convert_to_tensor(dedicom_torch.global_interaction.detach().numpy()) + for i in range(dedicom_tf.num_types): + dedicom_tf.vars['local_variation_%d' % i] = \ + tf.convert_to_tensor(dedicom_torch.local_variation[i].detach().numpy()) + + inputs = np.random.rand(20, 10).astype(np.float32) + inputs_torch = torch.tensor(inputs) + inputs_tf = { + 0: tf.convert_to_tensor(inputs) + } + out_torch = dedicom_torch(inputs_torch, inputs_torch) + out_tf = dedicom_tf(inputs_tf) + + assert len(out_torch) == len(out_tf) + assert len(out_tf) == 7 + + for i in range(len(out_torch)): + assert out_torch[i].shape == out_tf[i].shape + + sess = tf.Session() + for i in range(len(out_torch)): + item_torch = out_torch[i].detach().numpy() + item_tf = out_tf[i].eval(session=sess) + print('item_torch:', item_torch) + print('item_tf:', item_tf) + assert np.all(item_torch - item_tf < .000001) + sess.close() -- 2.26.2 From c4aefc015039e10fc3ecaa90d8ac276fbcd123a4 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 13 May 2020 18:25:40 +0200 Subject: [PATCH 010/227] Added DistMultDecoder with test. --- src/decagon_pytorch/decode.py | 31 ++++++++++++++++++- tests/decagon_pytorch/test_decode.py | 45 +++++++++++++++++++--------- 2 files changed, 61 insertions(+), 15 deletions(-) diff --git a/src/decagon_pytorch/decode.py b/src/decagon_pytorch/decode.py index 8ff6096..3643652 100644 --- a/src/decagon_pytorch/decode.py +++ b/src/decagon_pytorch/decode.py @@ -14,7 +14,6 @@ class DEDICOMDecoder(torch.nn.Module): self.drop_prob = drop_prob self.activation = activation - self.global_interaction = init_glorot(input_dim, input_dim) self.local_variation = [ torch.flatten(init_glorot(input_dim, 1)) \ @@ -35,3 +34,33 @@ class DEDICOMDecoder(torch.nn.Module): rec = torch.mm(product3, torch.transpose(inputs_col, 0, 1)) outputs.append(self.activation(rec)) return outputs + + +class DistMultDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, drop_prob=0., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.drop_prob = drop_prob + self.activation = activation + + self.relation = [ + torch.flatten(init_glorot(input_dim, 1)) \ + for _ in range(num_relation_types) + ] + + def forward(self, inputs_row, inputs_col): + outputs = [] + for k in range(self.num_relation_types): + inputs_row = dropout(inputs_row, 1.-self.drop_prob) + inputs_col = dropout(inputs_col, 1.-self.drop_prob) + + relation = torch.diag(self.relation[k]) + + intermediate_product = torch.mm(inputs_row, relation) + rec = torch.mm(intermediate_product, torch.transpose(inputs_col, 0, 1)) + outputs.append(self.activation(rec)) + return outputs diff --git a/tests/decagon_pytorch/test_decode.py b/tests/decagon_pytorch/test_decode.py index 7e8c4ed..a374480 100644 --- a/tests/decagon_pytorch/test_decode.py +++ b/tests/decagon_pytorch/test_decode.py @@ -5,25 +5,14 @@ import tensorflow as tf import torch -def test_dedicom(): - dedicom_torch = decagon_pytorch.decode.DEDICOMDecoder(input_dim=10, - num_relation_types=7) - dedicom_tf = decagon.deep.layers.DEDICOMDecoder(input_dim=10, num_types=7, - edge_type=(0, 0)) - - dedicom_tf.vars['global_interaction'] = \ - tf.convert_to_tensor(dedicom_torch.global_interaction.detach().numpy()) - for i in range(dedicom_tf.num_types): - dedicom_tf.vars['local_variation_%d' % i] = \ - tf.convert_to_tensor(dedicom_torch.local_variation[i].detach().numpy()) - +def _common(decoder_torch, decoder_tf): inputs = np.random.rand(20, 10).astype(np.float32) inputs_torch = torch.tensor(inputs) inputs_tf = { 0: tf.convert_to_tensor(inputs) } - out_torch = dedicom_torch(inputs_torch, inputs_torch) - out_tf = dedicom_tf(inputs_tf) + out_torch = decoder_torch(inputs_torch, inputs_torch) + out_tf = decoder_tf(inputs_tf) assert len(out_torch) == len(out_tf) assert len(out_tf) == 7 @@ -39,3 +28,31 @@ def test_dedicom(): print('item_tf:', item_tf) assert np.all(item_torch - item_tf < .000001) sess.close() + + +def test_dedicom_decoder(): + dedicom_torch = decagon_pytorch.decode.DEDICOMDecoder(input_dim=10, + num_relation_types=7) + dedicom_tf = decagon.deep.layers.DEDICOMDecoder(input_dim=10, num_types=7, + edge_type=(0, 0)) + + dedicom_tf.vars['global_interaction'] = \ + tf.convert_to_tensor(dedicom_torch.global_interaction.detach().numpy()) + for i in range(dedicom_tf.num_types): + dedicom_tf.vars['local_variation_%d' % i] = \ + tf.convert_to_tensor(dedicom_torch.local_variation[i].detach().numpy()) + + _common(dedicom_torch, dedicom_tf) + + +def test_dist_mult_decoder(): + distmult_torch = decagon_pytorch.decode.DistMultDecoder(input_dim=10, + num_relation_types=7) + distmult_tf = decagon.deep.layers.DistMultDecoder(input_dim=10, num_types=7, + edge_type=(0, 0)) + + for i in range(distmult_tf.num_types): + distmult_tf.vars['relation_%d' % i] = \ + tf.convert_to_tensor(distmult_torch.relation[i].detach().numpy()) + + _common(distmult_torch, distmult_tf) -- 2.26.2 From 438ba67565ccf4ff3223119f966e047820d9adf5 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 14 May 2020 19:02:08 +0200 Subject: [PATCH 011/227] Add BilinearDecoder and test. --- src/decagon_pytorch/decode.py | 28 ++++++++++++++++++++++++++++ tests/decagon_pytorch/test_decode.py | 13 +++++++++++++ 2 files changed, 41 insertions(+) diff --git a/src/decagon_pytorch/decode.py b/src/decagon_pytorch/decode.py index 3643652..8538c52 100644 --- a/src/decagon_pytorch/decode.py +++ b/src/decagon_pytorch/decode.py @@ -64,3 +64,31 @@ class DistMultDecoder(torch.nn.Module): rec = torch.mm(intermediate_product, torch.transpose(inputs_col, 0, 1)) outputs.append(self.activation(rec)) return outputs + + +class BilinearDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, drop_prob=0., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.drop_prob = drop_prob + self.activation = activation + + self.relation = [ + init_glorot(input_dim, input_dim) \ + for _ in range(num_relation_types) + ] + + def forward(self, inputs_row, inputs_col): + outputs = [] + for k in range(self.num_relation_types): + inputs_row = dropout(inputs_row, 1.-self.drop_prob) + inputs_col = dropout(inputs_col, 1.-self.drop_prob) + + intermediate_product = torch.mm(inputs_row, self.relation[k]) + rec = torch.mm(intermediate_product, torch.transpose(inputs_col, 0, 1)) + outputs.append(self.activation(rec)) + return outputs diff --git a/tests/decagon_pytorch/test_decode.py b/tests/decagon_pytorch/test_decode.py index a374480..198c6fe 100644 --- a/tests/decagon_pytorch/test_decode.py +++ b/tests/decagon_pytorch/test_decode.py @@ -56,3 +56,16 @@ def test_dist_mult_decoder(): tf.convert_to_tensor(distmult_torch.relation[i].detach().numpy()) _common(distmult_torch, distmult_tf) + + +def test_bilinear_decoder(): + bilinear_torch = decagon_pytorch.decode.BilinearDecoder(input_dim=10, + num_relation_types=7) + bilinear_tf = decagon.deep.layers.BilinearDecoder(input_dim=10, num_types=7, + edge_type=(0, 0)) + + for i in range(bilinear_tf.num_types): + bilinear_tf.vars['relation_%d' % i] = \ + tf.convert_to_tensor(bilinear_torch.relation[i].detach().numpy()) + + _common(bilinear_torch, bilinear_tf) -- 2.26.2 From e26ccd4222be2e4259a89b912c53ea4eb7fb3632 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 14 May 2020 19:19:03 +0200 Subject: [PATCH 012/227] Added InnerProductDecoder and test. --- src/decagon_pytorch/decode.py | 23 +++++++++++++++++++++++ tests/decagon_pytorch/test_decode.py | 9 +++++++++ 2 files changed, 32 insertions(+) diff --git a/src/decagon_pytorch/decode.py b/src/decagon_pytorch/decode.py index 8538c52..fded66c 100644 --- a/src/decagon_pytorch/decode.py +++ b/src/decagon_pytorch/decode.py @@ -92,3 +92,26 @@ class BilinearDecoder(torch.nn.Module): rec = torch.mm(intermediate_product, torch.transpose(inputs_col, 0, 1)) outputs.append(self.activation(rec)) return outputs + + +class InnerProductDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, drop_prob=0., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.drop_prob = drop_prob + self.activation = activation + + + def forward(self, inputs_row, inputs_col): + outputs = [] + for k in range(self.num_relation_types): + inputs_row = dropout(inputs_row, 1.-self.drop_prob) + inputs_col = dropout(inputs_col, 1.-self.drop_prob) + + rec = torch.mm(inputs_row, torch.transpose(inputs_col, 0, 1)) + outputs.append(self.activation(rec)) + return outputs diff --git a/tests/decagon_pytorch/test_decode.py b/tests/decagon_pytorch/test_decode.py index 198c6fe..798fc50 100644 --- a/tests/decagon_pytorch/test_decode.py +++ b/tests/decagon_pytorch/test_decode.py @@ -69,3 +69,12 @@ def test_bilinear_decoder(): tf.convert_to_tensor(bilinear_torch.relation[i].detach().numpy()) _common(bilinear_torch, bilinear_tf) + + +def test_inner_product_decoder(): + inner_torch = decagon_pytorch.decode.InnerProductDecoder(input_dim=10, + num_relation_types=7) + inner_tf = decagon.deep.layers.InnerProductDecoder(input_dim=10, num_types=7, + edge_type=(0, 0)) + + _common(inner_torch, inner_tf) -- 2.26.2 From ecabdc0540702748bbd3d0e2db2b453e10b4a162 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 15 May 2020 11:27:22 +0200 Subject: [PATCH 013/227] Add simple class to hold graph data. --- src/decagon_pytorch/batch.py | 0 src/decagon_pytorch/data.py | 38 ++++++++++++++++++++++++++++++ src/decagon_pytorch/normalize.py | 18 ++++++++++++++ tests/decagon_pytorch/test_data.py | 13 ++++++++++ 4 files changed, 69 insertions(+) create mode 100644 src/decagon_pytorch/batch.py create mode 100644 src/decagon_pytorch/data.py create mode 100644 src/decagon_pytorch/normalize.py create mode 100644 tests/decagon_pytorch/test_data.py diff --git a/src/decagon_pytorch/batch.py b/src/decagon_pytorch/batch.py new file mode 100644 index 0000000..e69de29 diff --git a/src/decagon_pytorch/data.py b/src/decagon_pytorch/data.py new file mode 100644 index 0000000..736f558 --- /dev/null +++ b/src/decagon_pytorch/data.py @@ -0,0 +1,38 @@ +class Data(object): + def __init__(self): + self.node_types = [] + self.relation_types = [] + + def add_node_type(self, name): + self.node_types.append(name) + + def add_relation(self, node_type_row, node_type_column, adjacency_matrix, name): + n = len(self.node_types) + if node_type_row >= n or node_type_column >= n: + raise ValueError('Node type index out of bounds, add node type first') + self.relation_types.append((node_type_row, node_type_column, adjacency_matrix, name)) + + def __repr__(self): + n = len(self.node_types) + if n == 0: + return 'Empty GNN Data' + s = '' + s += 'GNN Data with:\n' + s += '- ' + str(n) + ' node type(s):\n' + for nt in self.node_types: + s += ' - ' + nt + '\n' + if len(self.relation_types) == 0: + s += '- No relation types\n' + return s.strip() + s += '- ' + str(len(self.relation_types)) + ' relation type(s):\n' + for i in range(n): + for j in range(n): + rels = list(filter(lambda a: a[0] == i and a[1] == j, self.relation_types)) + if len(rels) == 0: + continue + # dir = '<->' if i == j else '->' + dir = '--' + s += ' - ' + self.node_types[i] + ' ' + dir + ' ' + self.node_types[j] + ':\n' + for r in rels: + s += ' - ' + r[3] + '\n' + return s.strip() diff --git a/src/decagon_pytorch/normalize.py b/src/decagon_pytorch/normalize.py new file mode 100644 index 0000000..54ddb8e --- /dev/null +++ b/src/decagon_pytorch/normalize.py @@ -0,0 +1,18 @@ +import numpy as np +import scipy.sparse as sp + + +def normalize_adjacency_matrix(self, adj): + adj = sp.coo_matrix(adj) + if adj.shape[0] == adj.shape[1]: + adj_ = adj + sp.eye(adj.shape[0]) + rowsum = np.array(adj_.sum(1)) + degree_mat_inv_sqrt = sp.diags(np.power(rowsum, -0.5).flatten()) + adj_normalized = adj_.dot(degree_mat_inv_sqrt).transpose().dot(degree_mat_inv_sqrt).tocoo() + else: + rowsum = np.array(adj.sum(1)) + colsum = np.array(adj.sum(0)) + rowdegree_mat_inv = sp.diags(np.nan_to_num(np.power(rowsum, -0.5)).flatten()) + coldegree_mat_inv = sp.diags(np.nan_to_num(np.power(colsum, -0.5)).flatten()) + adj_normalized = rowdegree_mat_inv.dot(adj).dot(coldegree_mat_inv).tocoo() + return preprocessing.sparse_to_tuple(adj_normalized) diff --git a/tests/decagon_pytorch/test_data.py b/tests/decagon_pytorch/test_data.py new file mode 100644 index 0000000..b0c2b56 --- /dev/null +++ b/tests/decagon_pytorch/test_data.py @@ -0,0 +1,13 @@ +from decagon_pytorch.data import Data + + +def test_data(): + d = Data() + d.add_node_type('Gene') + d.add_node_type('Drug') + d.add_relation(1, 0, None, 'Target') + d.add_relation(0, 0, None, 'Interaction') + d.add_relation(1, 1, None, 'Side Effect: Nausea') + d.add_relation(1, 1, None, 'Side Effect: Infertility') + d.add_relation(1, 1, None, 'Side Effect: Death') + print(d) -- 2.26.2 From 6b353878e197656e04398ccd629e50e043867806 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 15 May 2020 12:17:35 +0200 Subject: [PATCH 014/227] test_normalize --- src/decagon_pytorch/normalize.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/decagon_pytorch/normalize.py b/src/decagon_pytorch/normalize.py index 54ddb8e..577ad0d 100644 --- a/src/decagon_pytorch/normalize.py +++ b/src/decagon_pytorch/normalize.py @@ -2,17 +2,28 @@ import numpy as np import scipy.sparse as sp -def normalize_adjacency_matrix(self, adj): +def sparse_to_tuple(sparse_mx): + if not sp.isspmatrix_coo(sparse_mx): + sparse_mx = sparse_mx.tocoo() + coords = np.vstack((sparse_mx.row, sparse_mx.col)).transpose() + values = sparse_mx.data + shape = sparse_mx.shape + return coords, values, shape + + +def normalize_adjacency_matrix(adj): adj = sp.coo_matrix(adj) + if adj.shape[0] == adj.shape[1]: adj_ = adj + sp.eye(adj.shape[0]) rowsum = np.array(adj_.sum(1)) - degree_mat_inv_sqrt = sp.diags(np.power(rowsum, -0.5).flatten()) - adj_normalized = adj_.dot(degree_mat_inv_sqrt).transpose().dot(degree_mat_inv_sqrt).tocoo() + degree_mat_inv_sqrt = np.power(rowsum, -0.5).flatten() + degree_mat_inv_sqrt = sp.diags(degree_mat_inv_sqrt) + adj_normalized = adj_.dot(degree_mat_inv_sqrt).transpose().dot(degree_mat_inv_sqrt) else: rowsum = np.array(adj.sum(1)) colsum = np.array(adj.sum(0)) rowdegree_mat_inv = sp.diags(np.nan_to_num(np.power(rowsum, -0.5)).flatten()) coldegree_mat_inv = sp.diags(np.nan_to_num(np.power(colsum, -0.5)).flatten()) adj_normalized = rowdegree_mat_inv.dot(adj).dot(coldegree_mat_inv).tocoo() - return preprocessing.sparse_to_tuple(adj_normalized) + return sparse_to_tuple(adj_normalized) -- 2.26.2 From d835b049b029fc27c7bd2d04b52665d28ee2438c Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 15 May 2020 12:19:29 +0200 Subject: [PATCH 015/227] test_normalize --- tests/decagon_pytorch/test_normalize.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/decagon_pytorch/test_normalize.py diff --git a/tests/decagon_pytorch/test_normalize.py b/tests/decagon_pytorch/test_normalize.py new file mode 100644 index 0000000..c7e0180 --- /dev/null +++ b/tests/decagon_pytorch/test_normalize.py @@ -0,0 +1,25 @@ +import decagon_pytorch.normalize +import decagon.deep.minibatch +import numpy as np + + +def test_normalize_adjacency_matrix_square(): + mx = np.random.rand(10, 10) + mx[mx < .5] = 0 + mx = np.ceil(mx) + res_torch = decagon_pytorch.normalize.normalize_adjacency_matrix(mx) + res_tf = decagon.deep.minibatch.EdgeMinibatchIterator.preprocess_graph(None, mx) + assert len(res_torch) == len(res_tf) + for i in range(len(res_torch)): + assert np.all(res_torch[i] == res_tf[i]) + + +def test_normalize_adjacency_matrix_nonsquare(): + mx = np.random.rand(5, 10) + mx[mx < .5] = 0 + mx = np.ceil(mx) + res_torch = decagon_pytorch.normalize.normalize_adjacency_matrix(mx) + res_tf = decagon.deep.minibatch.EdgeMinibatchIterator.preprocess_graph(None, mx) + assert len(res_torch) == len(res_tf) + for i in range(len(res_torch)): + assert np.all(res_torch[i] == res_tf[i]) -- 2.26.2 From 71bf3384910ddc979c7d590b190745a8ad605e79 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 15 May 2020 13:52:29 +0200 Subject: [PATCH 016/227] Start implementing Minibatch. --- src/decagon_pytorch/batch.py | 43 ++++++++++++++++++++++++++++++ src/decagon_pytorch/data.py | 24 +++++++++++++++-- tests/decagon_pytorch/test_data.py | 2 ++ 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/decagon_pytorch/batch.py b/src/decagon_pytorch/batch.py index e69de29..8be3fe6 100644 --- a/src/decagon_pytorch/batch.py +++ b/src/decagon_pytorch/batch.py @@ -0,0 +1,43 @@ +import scipy.sparse as sp + + +class Batch(object): + def __init__(self, adjacency_matrix): + pass + + def get(size): + pass + + +def train_test_split(data, train_size=.8): + pass + + +class Minibatch(object): + def __init__(self, data, node_type_row, node_type_column, size): + self.data = data + self.adjacency_matrix = data.get_adjacency_matrix(node_type_row, node_type_column) + self.size = size + self.order = np.random.permutation(adjacency_matrix.nnz) + self.count = 0 + + def reset(self): + self.count = 0 + self.order = np.random.permutation(adjacency_matrix.nnz) + + def __iter__(self): + adj_mat = self.adjacency_matrix + size = self.size + order = np.random.permutation(adj_mat.nnz) + for i in range(0, len(order), size): + row = adj_mat.row[i:i + size] + col = adj_mat.col[i:i + size] + data = adj_mat.data[i:i + size] + adj_mat_batch = sp.coo_matrix((data, (row, col)), shape=adj_mat.shape) + yield adj_mat_batch + degree = self.adjacency_matrix.sum(1) + + + + def __len__(self): + pass diff --git a/src/decagon_pytorch/data.py b/src/decagon_pytorch/data.py index 736f558..3c62469 100644 --- a/src/decagon_pytorch/data.py +++ b/src/decagon_pytorch/data.py @@ -1,16 +1,36 @@ +from collections import defaultdict +from .decode import BilinearDecoder +from .weights import init_glorot + + class Data(object): def __init__(self): self.node_types = [] self.relation_types = [] + self.decoder_types = defaultdict(lambda: BilinearDecoder) + self.latent_node = [] - def add_node_type(self, name): + def add_node_type(self, name, count, latent_length): self.node_types.append(name) + self.latent_node.append(init_glorot(count, latent_length)) def add_relation(self, node_type_row, node_type_column, adjacency_matrix, name): n = len(self.node_types) if node_type_row >= n or node_type_column >= n: raise ValueError('Node type index out of bounds, add node type first') self.relation_types.append((node_type_row, node_type_column, adjacency_matrix, name)) + _ = self.decoder_types[(node_type_row, node_type_column)] + + def set_decoder_type(self, node_type_row, node_type_column, decoder_class): + if (node_type_row, node_type_column) not in self.decoder_types: + raise ValueError('Relation type not found, add relation first') + self.decoder_types[(node_type_row, node_type_column)] = decoder_class + + def get_adjacency_matrices(self, node_type_row, node_type_column): + rels = list(filter(lambda a: a[0] == node_type_row and a[1] == node_type_column), self.relation_types) + if len(rels) == 0: + + def __repr__(self): n = len(self.node_types) @@ -32,7 +52,7 @@ class Data(object): continue # dir = '<->' if i == j else '->' dir = '--' - s += ' - ' + self.node_types[i] + ' ' + dir + ' ' + self.node_types[j] + ':\n' + s += ' - ' + self.node_types[i] + ' ' + dir + ' ' + self.node_types[j] + ' (' + self.decoder_types[(i, j)].__name__ + '):\n' for r in rels: s += ' - ' + r[3] + '\n' return s.strip() diff --git a/tests/decagon_pytorch/test_data.py b/tests/decagon_pytorch/test_data.py index b0c2b56..b37c14b 100644 --- a/tests/decagon_pytorch/test_data.py +++ b/tests/decagon_pytorch/test_data.py @@ -1,4 +1,5 @@ from decagon_pytorch.data import Data +from decagon_pytorch.decode import DEDICOMDecoder def test_data(): @@ -10,4 +11,5 @@ def test_data(): d.add_relation(1, 1, None, 'Side Effect: Nausea') d.add_relation(1, 1, None, 'Side Effect: Infertility') d.add_relation(1, 1, None, 'Side Effect: Death') + d.set_decoder_type(1, 1, DEDICOMDecoder) print(d) -- 2.26.2 From 21b256572073c7bdfeec62249298a2f99acc06c5 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 19 May 2020 14:22:17 +0200 Subject: [PATCH 017/227] Changed Data architecture a bit. --- src/decagon_pytorch/data.py | 70 ++++++++++++++++++++---------- src/decagon_pytorch/model.py | 6 +++ tests/decagon_pytorch/test_data.py | 15 +++---- 3 files changed, 61 insertions(+), 30 deletions(-) diff --git a/src/decagon_pytorch/data.py b/src/decagon_pytorch/data.py index 3c62469..0891c8b 100644 --- a/src/decagon_pytorch/data.py +++ b/src/decagon_pytorch/data.py @@ -3,33 +3,53 @@ from .decode import BilinearDecoder from .weights import init_glorot +class NodeType(object): + def __init__(self, name, count): + self.name = name + self.count = count + + +class RelationType(object): + def __init__(self, name, node_type_row, node_type_column, + adjacency_matrix): + self.name = name + self.node_type_row = node_type_row + self.node_type_column = node_type_column + self.adjacency_matrix = adjacency_matrix + + class Data(object): def __init__(self): self.node_types = [] - self.relation_types = [] - self.decoder_types = defaultdict(lambda: BilinearDecoder) - self.latent_node = [] + self.relation_types = defaultdict(list) + # self.decoder_types = defaultdict(lambda: BilinearDecoder) + # self.latent_node = [] - def add_node_type(self, name, count, latent_length): - self.node_types.append(name) - self.latent_node.append(init_glorot(count, latent_length)) + def add_node_type(self, name, count): # , latent_length): + self.node_types.append(NodeType(name, count)) + # self.latent_node.append(init_glorot(count, latent_length)) - def add_relation(self, node_type_row, node_type_column, adjacency_matrix, name): + def add_relation_type(self, name, node_type_row, node_type_column, adjacency_matrix): n = len(self.node_types) if node_type_row >= n or node_type_column >= n: raise ValueError('Node type index out of bounds, add node type first') - self.relation_types.append((node_type_row, node_type_column, adjacency_matrix, name)) - _ = self.decoder_types[(node_type_row, node_type_column)] + key = (node_type_row, node_type_column) + self.relation_types[key].append(RelationType(name, node_type_row, node_type_column, adjacency_matrix)) + # _ = self.decoder_types[(node_type_row, node_type_column)] - def set_decoder_type(self, node_type_row, node_type_column, decoder_class): - if (node_type_row, node_type_column) not in self.decoder_types: - raise ValueError('Relation type not found, add relation first') - self.decoder_types[(node_type_row, node_type_column)] = decoder_class + #def set_decoder_type(self, node_type_row, node_type_column, decoder_class): + # if (node_type_row, node_type_column) not in self.decoder_types: + # raise ValueError('Relation type not found, add relation first') + # self.decoder_types[(node_type_row, node_type_column)] = decoder_class def get_adjacency_matrices(self, node_type_row, node_type_column): - rels = list(filter(lambda a: a[0] == node_type_row and a[1] == node_type_column), self.relation_types) - if len(rels) == 0: - + # rels = list(filter(lambda a: a[0] == node_type_row and a[1] == node_type_column), self.relation_types) + key = (node_type_row, node_type_column) + if key not in self.relation_types: + raise ValueError('Relation type not found') + rels = self.relation_types[key] + rels = list(map(lambda a: a.adjacency_matrix, rels)) + return rels def __repr__(self): @@ -40,19 +60,25 @@ class Data(object): s += 'GNN Data with:\n' s += '- ' + str(n) + ' node type(s):\n' for nt in self.node_types: - s += ' - ' + nt + '\n' + s += ' - ' + nt.name + '\n' if len(self.relation_types) == 0: s += '- No relation types\n' return s.strip() - s += '- ' + str(len(self.relation_types)) + ' relation type(s):\n' + n = sum(map(len, self.relation_types)) + s += '- ' + str(n) + ' relation type(s):\n' for i in range(n): for j in range(n): - rels = list(filter(lambda a: a[0] == i and a[1] == j, self.relation_types)) - if len(rels) == 0: + key = (i, j) + if key not in self.relation_types: continue + rels = self.relation_types[key] + # rels = list(filter(lambda a: a[0] == i and a[1] == j, self.relation_types)) + #if len(rels) == 0: + # continue # dir = '<->' if i == j else '->' dir = '--' - s += ' - ' + self.node_types[i] + ' ' + dir + ' ' + self.node_types[j] + ' (' + self.decoder_types[(i, j)].__name__ + '):\n' + s += ' - ' + self.node_types[i].name + ' ' + dir + ' ' + self.node_types[j].name + ':\n' + #' (' + self.decoder_types[(i, j)].__name__ + '):\n' for r in rels: - s += ' - ' + r[3] + '\n' + s += ' - ' + r.name + '\n' return s.strip() diff --git a/src/decagon_pytorch/model.py b/src/decagon_pytorch/model.py index e69de29..0e6380b 100644 --- a/src/decagon_pytorch/model.py +++ b/src/decagon_pytorch/model.py @@ -0,0 +1,6 @@ +class Model(object): + def __init__(self, data): + self.data = data + + def build(self): + pass diff --git a/tests/decagon_pytorch/test_data.py b/tests/decagon_pytorch/test_data.py index b37c14b..fc6c111 100644 --- a/tests/decagon_pytorch/test_data.py +++ b/tests/decagon_pytorch/test_data.py @@ -4,12 +4,11 @@ from decagon_pytorch.decode import DEDICOMDecoder def test_data(): d = Data() - d.add_node_type('Gene') - d.add_node_type('Drug') - d.add_relation(1, 0, None, 'Target') - d.add_relation(0, 0, None, 'Interaction') - d.add_relation(1, 1, None, 'Side Effect: Nausea') - d.add_relation(1, 1, None, 'Side Effect: Infertility') - d.add_relation(1, 1, None, 'Side Effect: Death') - d.set_decoder_type(1, 1, DEDICOMDecoder) + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + d.add_relation_type('Target', 1, 0, None) + d.add_relation_type('Interaction', 0, 0, None) + d.add_relation_type('Side Effect: Nausea', 1, 1, None) + d.add_relation_type('Side Effect: Infertility', 1, 1, None) + d.add_relation_type('Side Effect: Death', 1, 1, None) print(d) -- 2.26.2 From 5464c5a4c1fe01c4e04b1fae6b402c54f0248924 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 19 May 2020 15:28:35 +0200 Subject: [PATCH 018/227] Add InputLayer. --- src/decagon_pytorch/layer.py | 60 +++++++++++++++++++++++++++++ tests/decagon_pytorch/test_data.py | 1 - tests/decagon_pytorch/test_layer.py | 52 +++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 src/decagon_pytorch/layer.py create mode 100644 tests/decagon_pytorch/test_layer.py diff --git a/src/decagon_pytorch/layer.py b/src/decagon_pytorch/layer.py new file mode 100644 index 0000000..3929062 --- /dev/null +++ b/src/decagon_pytorch/layer.py @@ -0,0 +1,60 @@ +# +# This module implements a single layer of the Decagon +# model. This is going to be already quite complex, as +# we will be using all the graph convolutional building +# blocks. +# +# h_{i}^(k+1) = ϕ(∑_r ∑_{j∈N{r}^{i}} c_{r}^{ij} * \ +# W_{r}^(k) h_{j}^{k} + c_{r}^{i} h_{i}^(k)) +# +# N{r}^{i} - set of neighbors of node i under relation r +# W_{r}^(k) - relation-type specific weight matrix +# h_{i}^(k) - hidden state of node i in layer k +# h_{i}^(k)∈R^{d(k)} where d(k) is the dimensionality +# of the representation in k-th layer +# ϕ - activation function +# c_{r}^{ij} - normalization constants +# c_{r}^{ij} = 1/sqrt(|N_{r}^{i}| |N_{r}^{j}|) +# c_{r}^{i} - normalization constants +# c_{r}^{i} = 1/|N_{r}^{i}| +# + + +import torch + + +class InputLayer(torch.nn.Module): + def __init__(self, data, dimensionality=32, **kwargs): + super().__init__(**kwargs) + self.data = data + self.dimensionality = dimensionality + self.node_reps = None + self.build() + + def build(self): + self.node_reps = [] + for i, nt in enumerate(self.data.node_types): + reps = torch.rand(nt.count, self.dimensionality) + reps = torch.nn.Parameter(reps) + self.register_parameter('node_reps[%d]' % i, reps) + self.node_reps.append(reps) + + def forward(self): + return self.node_reps + + def __repr__(self): + s = '' + s += 'GNN input layer with dimensionality: %d\n' % self.dimensionality + s += ' # of node types: %d\n' % len(self.data.node_types) + for nt in self.data.node_types: + s += ' - %s (%d)\n' % (nt.name, nt.count) + return s.strip() + + +class DecagonLayer(torch.nn.Module): + def __init__(self, data, **kwargs): + super().__init__(**kwargs) + self.data = data + + def __call__(self, previous_layer): + pass diff --git a/tests/decagon_pytorch/test_data.py b/tests/decagon_pytorch/test_data.py index fc6c111..51426ee 100644 --- a/tests/decagon_pytorch/test_data.py +++ b/tests/decagon_pytorch/test_data.py @@ -1,5 +1,4 @@ from decagon_pytorch.data import Data -from decagon_pytorch.decode import DEDICOMDecoder def test_data(): diff --git a/tests/decagon_pytorch/test_layer.py b/tests/decagon_pytorch/test_layer.py new file mode 100644 index 0000000..7e2115c --- /dev/null +++ b/tests/decagon_pytorch/test_layer.py @@ -0,0 +1,52 @@ +from decagon_pytorch.layer import InputLayer +from decagon_pytorch.data import Data +import torch +import pytest + + +def _some_data(): + d = Data() + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + d.add_relation_type('Target', 1, 0, None) + d.add_relation_type('Interaction', 0, 0, None) + d.add_relation_type('Side Effect: Nausea', 1, 1, None) + d.add_relation_type('Side Effect: Infertility', 1, 1, None) + d.add_relation_type('Side Effect: Death', 1, 1, None) + return d + + +def test_input_layer_01(): + d = _some_data() + for dimensionality in [32, 64, 128]: + layer = InputLayer(d, dimensionality) + assert layer.dimensionality == dimensionality + assert len(layer.node_reps) == 2 + assert layer.node_reps[0].shape == (1000, dimensionality) + assert layer.node_reps[1].shape == (100, dimensionality) + assert layer.data == d + + +def test_input_layer_02(): + d = _some_data() + layer = InputLayer(d, 32) + res = layer() + assert isinstance(res[0], torch.Tensor) + assert isinstance(res[1], torch.Tensor) + assert res[0].shape == (1000, 32) + assert res[1].shape == (100, 32) + assert torch.all(res[0] == layer.node_reps[0]) + assert torch.all(res[1] == layer.node_reps[1]) + + +def test_input_layer_03(): + if torch.cuda.device_count() == 0: + pytest.skip('No CUDA devices on this host') + d = _some_data() + layer = InputLayer(d, 32) + device = torch.device('cuda:0') + layer = layer.to(device) + print(list(layer.parameters())) + # assert layer.device.type == 'cuda:0' + assert layer.node_reps[0].device == device + assert layer.node_reps[1].device == device -- 2.26.2 From f398b72267de9281b29034104e812fc61044b6cd Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 20 May 2020 16:52:39 +0200 Subject: [PATCH 019/227] Start implementing DecagonLayer. --- src/decagon_pytorch/layer.py | 46 +++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/src/decagon_pytorch/layer.py b/src/decagon_pytorch/layer.py index 3929062..97c7c2a 100644 --- a/src/decagon_pytorch/layer.py +++ b/src/decagon_pytorch/layer.py @@ -21,6 +21,7 @@ import torch +from .convole import SparseMultiDGCA class InputLayer(torch.nn.Module): @@ -52,9 +53,48 @@ class InputLayer(torch.nn.Module): class DecagonLayer(torch.nn.Module): - def __init__(self, data, **kwargs): + def __init__(self, data, + input_dim, output_dim, + keep_prob=1., + rel_activation=lambda x: x, + layer_activation=torch.nn.functional.relu, + **kwargs): super().__init__(**kwargs) self.data = data + self.input_dim = input_dim + self.output_dim = output_dim + self.keep_prob = keep_prob + self.rel_activation = rel_activation + self.layer_activation = layer_activation + self.convolutions = None + self.build() + + def build(self): + self.convolutions = {} + for key in self.data.relation_types.keys(): + adjacency_matrices = \ + self.data.get_adjacency_matrices(*key) + self.convolutions[key] = SparseMultiDGCA(self.input_dim, + self.output_dim, adjacency_matrices, + self.keep_prob, self.rel_activation) - def __call__(self, previous_layer): - pass + # for node_type_row, node_type_col in enumerate(self.data.node_ + # if rt.node_type_row == i or rt.node_type_col == i: + + def __call__(self, prev_layer_repr): + new_layer_repr = [] + for i, nt in enumerate(self.data.node_types): + new_repr = [] + for key in self.data.relation_types.keys(): + nt_row, nt_col = key + if nt_row != i and nt_col != i: + continue + if nt_row == i: + x = prev_layer_repr[nt_col] + else: + x = prev_layer_repr[nt_row] + conv = self.convolutions[key] + new_repr.append(conv(x)) + new_repr = sum(new_repr) + new_layer_repr.append(new_repr) + return new_layer_repr -- 2.26.2 From 46c2c676ba3a391cc69ce5175833312d51bcae5a Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 22 May 2020 13:49:09 +0200 Subject: [PATCH 020/227] Add type annotations to convolve. --- src/decagon_pytorch/convolve.py | 376 +++++++++++++++++++++++--------- src/decagon_pytorch/layer.py | 2 +- 2 files changed, 273 insertions(+), 105 deletions(-) diff --git a/src/decagon_pytorch/convolve.py b/src/decagon_pytorch/convolve.py index 3b116e7..d50f4b6 100644 --- a/src/decagon_pytorch/convolve.py +++ b/src/decagon_pytorch/convolve.py @@ -1,104 +1,272 @@ -import torch -from .dropout import dropout_sparse, \ - dropout -from .weights import init_glorot - - -class SparseGraphConv(torch.nn.Module): - """Convolution layer for sparse inputs.""" - def __init__(self, in_channels, out_channels, - adjacency_matrix, **kwargs): - super().__init__(**kwargs) - self.in_channels = in_channels - self.out_channels = out_channels - self.weight = init_glorot(in_channels, out_channels) - self.adjacency_matrix = adjacency_matrix - - - def forward(self, x): - x = torch.sparse.mm(x, self.weight) - x = torch.sparse.mm(self.adjacency_matrix, x) - return x - - -class SparseDropoutGraphConvActivation(torch.nn.Module): - def __init__(self, input_dim, output_dim, - adjacency_matrix, keep_prob=1., - activation=torch.nn.functional.relu, - **kwargs): - super().__init__(**kwargs) - self.sparse_graph_conv = SparseGraphConv(input_dim, output_dim, adjacency_matrix) - self.keep_prob = keep_prob - self.activation = activation - - def forward(self, x): - x = dropout_sparse(x, self.keep_prob) - x = self.sparse_graph_conv(x) - x = self.activation(x) - return x - - -class SparseMultiDGCA(torch.nn.Module): - def __init__(self, input_dim, output_dim, - adjacency_matrices, keep_prob=1., - activation=torch.nn.functional.relu, - **kwargs): - super().__init__(**kwargs) - self.output_dim = output_dim - self.sparse_dgca = [ SparseDropoutGraphConvActivation(input_dim, output_dim, adj_mat, keep_prob, activation) for adj_mat in adjacency_matrices ] - - def forward(self, x): - out = torch.zeros(len(x), self.output_dim, dtype=x.dtype) - for f in self.sparse_dgca: - out += f(x) - out = torch.nn.functional.normalize(out, p=2, dim=1) - return out - - -class GraphConv(torch.nn.Module): - def __init__(self, in_channels, out_channels, - adjacency_matrix, **kwargs): - super().__init__(**kwargs) - self.in_channels = in_channels - self.out_channels = out_channels - self.weight = init_glorot(in_channels, out_channels) - self.adjacency_matrix = adjacency_matrix - - def forward(self, x): - x = torch.mm(x, self.weight) - x = torch.mm(self.adjacency_matrix, x) - return x - - -class DropoutGraphConvActivation(torch.nn.Module): - def __init__(self, input_dim, output_dim, - adjacency_matrix, keep_prob=1., - activation=torch.nn.functional.relu, - **kwargs): - super().__init__(**kwargs) - self.graph_conv = GraphConv(input_dim, output_dim, adjacency_matrix) - self.keep_prob = keep_prob - self.activation = activation - - def forward(self, x): - x = dropout(x, keep_prob=self.keep_prob) - x = self.graph_conv(x) - x = self.activation(x) - return x - - -class MultiDGCA(torch.nn.Module): - def __init__(self, input_dim, output_dim, - adjacency_matrices, keep_prob=1., - activation=torch.nn.functional.relu, - **kwargs): - super().__init__(**kwargs) - self.output_dim = output_dim - self.dgca = [ DropoutGraphConvActivation(input_dim, output_dim, adj_mat, keep_prob, activation) for adj_mat in adjacency_matrices ] - - def forward(self, x): - out = torch.zeros(len(x), self.output_dim, dtype=x.dtype) - for f in self.dgca: - out += f(x) - out = torch.nn.functional.normalize(out, p=2, dim=1) - return out +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + +""" +This module implements the basic convolutional blocks of Decagon. +Just as a quick reminder, the basic convolution formula here is: + +y = A * (x * W) + +where: + +W is a weight matrix +A is an adjacency matrix +x is a matrix of latent representations of a particular type of neighbors. + +As we have x here twice, a trick is obviously necessary for this to work. +A must be previously normalized with: + +c_{r}^{ij} = 1/sqrt(|N_{r}^{i}| |N_{r}^{j}|) + +or + +c_{r}^{i} = 1/|N_{r}^{i}| + +Let's work through this step by step to convince ourselves that the +formula is correct. + +x = [ + [0, 1, 0, 1], + [1, 1, 1, 0], + [0, 0, 0, 1] +] + +W = [ + [0, 1], + [1, 0], + [0.5, 0.5], + [0.25, 0.75] +] + +A = [ + [0, 1, 0], + [1, 0, 1], + [0, 1, 0] +] + +so the graph looks like this: + +(0) -- (1) -- (2) + +and therefore the representations in the next layer should be: + +h_{0}^{k+1} = c_{r}^{0,1} * h_{1}^{k} * W + c_{r}^{0} * h_{0}^{k} +h_{1}^{k+1} = c_{r}^{0,1} * h_{0}^{k} * W + c_{r}^{2,1} * h_{2}^{k} + + c_{r}^{1} * h_{1}^{k} +h_{2}^{k+1} = c_{r}^{2,1} * h_{1}^{k} * W + c_{r}^{2} * h_{2}^{k} + +In actual Decagon code we can see that that latter part propagating directly +the old representation is gone. I will try to do the same for now. + +So we have to only take care of: + +h_{0}^{k+1} = c_{r}^{0,1} * h_{1}^{k} * W +h_{1}^{k+1} = c_{r}^{0,1} * h_{0}^{k} * W + c_{r}^{2,1} * h_{2}^{k} +h_{2}^{k+1} = c_{r}^{2,1} * h_{1}^{k} * W + +If A is square the Decagon's EdgeMinibatchIterator preprocesses it as follows: + +A = A + eye(len(A)) +rowsum = A.sum(1) +deg_mat_inv_sqrt = diags(power(rowsum, -0.5)) +A = dot(A, deg_mat_inv_sqrt) +A = A.transpose() +A = A.dot(deg_mat_inv_sqrt) + +Let's see what gives in our case: + +A = A + eye(len(A)) + +[ + [1, 1, 0], + [1, 1, 1], + [0, 1, 1] +] + +rowsum = A.sum(1) + +[2, 3, 2] + +deg_mat_inv_sqrt = diags(power(rowsum, -0.5)) + +[ + [1./sqrt(2), 0, 0], + [0, 1./sqrt(3), 0], + [0, 0, 1./sqrt(2)] +] + +A = dot(A, deg_mat_inv_sqrt) + +[ + [ 1/sqrt(2), 1/sqrt(3), 0 ], + [ 1/sqrt(2), 1/sqrt(3), 1/sqrt(2) ], + [ 0, 1/sqrt(3), 1/sqrt(2) ] +] + +A = A.transpose() + +[ + [ 1/sqrt(2), 1/sqrt(2), 0 ], + [ 1/sqrt(3), 1/sqrt(3), 1/sqrt(3) ], + [ 0, 1/sqrt(2), 1/sqrt(2) ] +] + +A = A.dot(deg_mat_inv_sqrt) + +[ + [ 1/sqrt(2) * 1/sqrt(2), 1/sqrt(2) * 1/sqrt(3), 0 ], + [ 1/sqrt(3) * 1/sqrt(2), 1/sqrt(3) * 1/sqrt(3), 1/sqrt(3) * 1/sqrt(2) ], + [ 0, 1/sqrt(2) * 1/sqrt(3), 1/sqrt(2) * 1/sqrt(2) ], +] + +thus: + +[ + [0.5 , 0.40824829, 0. ], + [0.40824829, 0.33333333, 0.40824829], + [0. , 0.40824829, 0.5 ] +] + +This checks out with the 1/sqrt(|N_{r}^{i}| |N_{r}^{j}|) formula. + +Then, we get back to the main calculation: + +y = x * W +y = A * y + +y = x * W + +[ + [ 1.25, 0.75 ], + [ 1.5 , 1.5 ], + [ 0.25, 0.75 ] +] + +y = A * y + +[ + 0.5 * [ 1.25, 0.75 ] + 0.40824829 * [ 1.5, 1.5 ], + 0.40824829 * [ 1.25, 0.75 ] + 0.33333333 * [ 1.5, 1.5 ] + 0.40824829 * [ 0.25, 0.75 ], + 0.40824829 * [ 1.5, 1.5 ] + 0.5 * [ 0.25, 0.75 ] +] + +that is: + +[ + [1.23737243, 0.98737244], + [1.11237243, 1.11237243], + [0.73737244, 0.98737244] +]. + +All checks out nicely, good. + +""" + + +import torch +from .dropout import dropout_sparse, \ + dropout +from .weights import init_glorot +from typing import List, Callable + + +class SparseGraphConv(torch.nn.Module): + """Convolution layer for sparse inputs.""" + def __init__(self, in_channels: int, out_channels: int, + adjacency_matrix: torch.Tensor, **kwargs) -> None: + super().__init__(**kwargs) + self.in_channels = in_channels + self.out_channels = out_channels + self.weight = init_glorot(in_channels, out_channels) + self.adjacency_matrix = adjacency_matrix + + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = torch.sparse.mm(x, self.weight) + x = torch.sparse.mm(self.adjacency_matrix, x) + return x + + +class SparseDropoutGraphConvActivation(torch.nn.Module): + def __init__(self, input_dim: int, output_dim: int, + adjacency_matrix: torch.Tensor, keep_prob: float=1., + activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, + **kwargs) -> None: + super().__init__(**kwargs) + self.sparse_graph_conv = SparseGraphConv(input_dim, output_dim, adjacency_matrix) + self.keep_prob = keep_prob + self.activation = activation + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = dropout_sparse(x, self.keep_prob) + x = self.sparse_graph_conv(x) + x = self.activation(x) + return x + + +class SparseMultiDGCA(torch.nn.Module): + def __init__(self, input_dim: List[int], output_dim: int, + adjacency_matrices: List[torch.Tensor], keep_prob: float=1., + activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, + **kwargs) -> None: + super().__init__(**kwargs) + self.output_dim = output_dim + self.sparse_dgca = [ SparseDropoutGraphConvActivation(input_dim, output_dim, adj_mat, keep_prob, activation) for adj_mat in adjacency_matrices ] + + def forward(self, x: List[torch.Tensor]) -> List[torch.Tensor]: + out = torch.zeros(len(x), self.output_dim, dtype=x.dtype) + for f in self.sparse_dgca: + out += f(x) + out = torch.nn.functional.normalize(out, p=2, dim=1) + return out + + +class GraphConv(torch.nn.Module): + def __init__(self, in_channels: int, out_channels: int, + adjacency_matrix: torch.Tensor, **kwargs) -> None: + super().__init__(**kwargs) + self.in_channels = in_channels + self.out_channels = out_channels + self.weight = init_glorot(in_channels, out_channels) + self.adjacency_matrix = adjacency_matrix + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = torch.mm(x, self.weight) + x = torch.mm(self.adjacency_matrix, x) + return x + + +class DropoutGraphConvActivation(torch.nn.Module): + def __init__(self, input_dim: int, output_dim: int, + adjacency_matrix: torch.Tensor, keep_prob: float=1., + activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, + **kwargs) -> None: + super().__init__(**kwargs) + self.graph_conv = GraphConv(input_dim, output_dim, adjacency_matrix) + self.keep_prob = keep_prob + self.activation = activation + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = dropout(x, keep_prob=self.keep_prob) + x = self.graph_conv(x) + x = self.activation(x) + return x + + +class MultiDGCA(torch.nn.Module): + def __init__(self, input_dim: List[int], output_dim: int, + adjacency_matrices: List[torch.Tensor], keep_prob: float=1., + activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, + **kwargs) -> None: + super().__init__(**kwargs) + self.output_dim = output_dim + self.dgca = [ DropoutGraphConvActivation(input_dim, output_dim, adj_mat, keep_prob, activation) for adj_mat in adjacency_matrices ] + + def forward(self, x: List[torch.Tensor]) -> List[torch.Tensor]: + out = torch.zeros(len(x), self.output_dim, dtype=x.dtype) + for f in self.dgca: + out += f(x) + out = torch.nn.functional.normalize(out, p=2, dim=1) + return out diff --git a/src/decagon_pytorch/layer.py b/src/decagon_pytorch/layer.py index 97c7c2a..a75c005 100644 --- a/src/decagon_pytorch/layer.py +++ b/src/decagon_pytorch/layer.py @@ -21,7 +21,7 @@ import torch -from .convole import SparseMultiDGCA +from .convolve import SparseMultiDGCA class InputLayer(torch.nn.Module): -- 2.26.2 From e371a833520cbedf702e08522733f7c066359ada Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 22 May 2020 14:19:53 +0200 Subject: [PATCH 021/227] Add support for multiple input_dim in SparseMultiDGCA. --- src/decagon_pytorch/convolve.py | 24 ++++++++++++++++++------ tests/decagon_pytorch/test_convolve.py | 8 ++++++-- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/decagon_pytorch/convolve.py b/src/decagon_pytorch/convolve.py index d50f4b6..4ff9ee8 100644 --- a/src/decagon_pytorch/convolve.py +++ b/src/decagon_pytorch/convolve.py @@ -212,13 +212,25 @@ class SparseMultiDGCA(torch.nn.Module): activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, **kwargs) -> None: super().__init__(**kwargs) + self.input_dim = input_dim self.output_dim = output_dim - self.sparse_dgca = [ SparseDropoutGraphConvActivation(input_dim, output_dim, adj_mat, keep_prob, activation) for adj_mat in adjacency_matrices ] - - def forward(self, x: List[torch.Tensor]) -> List[torch.Tensor]: - out = torch.zeros(len(x), self.output_dim, dtype=x.dtype) - for f in self.sparse_dgca: - out += f(x) + self.adjacency_matrices = adjacency_matrices + self.keep_prob = keep_prob + self.activation = activation + self.sparse_dgca = None + self.build() + + def build(self): + if len(self.input_dim) != len(self.adjacency_matrices): + raise ValueError('input_dim must have the same length as adjacency_matrices') + self.sparse_dgca = [] + for input_dim, adj_mat in zip(self.input_dim, self.adjacency_matrices): + self.sparse_dgca.append(SparseDropoutGraphConvActivation(input_dim, self.output_dim, adj_mat, self.keep_prob, self.activation)) + + def forward(self, x: List[torch.Tensor]) -> torch.Tensor: + out = torch.zeros(len(x[0]), self.output_dim, dtype=x[0].dtype) + for i, f in enumerate(self.sparse_dgca): + out += f(x[i]) out = torch.nn.functional.normalize(out, p=2, dim=1) return out diff --git a/tests/decagon_pytorch/test_convolve.py b/tests/decagon_pytorch/test_convolve.py index 210a4e5..71e1ece 100644 --- a/tests/decagon_pytorch/test_convolve.py +++ b/tests/decagon_pytorch/test_convolve.py @@ -236,16 +236,20 @@ def test_multi_dgca(): assert np.all(adjacency_matrices[i].numpy() == adjacency_matrices_sparse[i].to_dense().numpy()) torch.random.manual_seed(0) - multi_sparse = decagon_pytorch.convolve.SparseMultiDGCA(10, 10, adjacency_matrices_sparse, keep_prob=keep_prob) + multi_sparse = decagon_pytorch.convolve.SparseMultiDGCA([10,]*len(adjacency_matrices), 10, adjacency_matrices_sparse, keep_prob=keep_prob) torch.random.manual_seed(0) multi = decagon_pytorch.convolve.MultiDGCA(10, 10, adjacency_matrices, keep_prob=keep_prob) + print('len(adjacency_matrices):', len(adjacency_matrices)) + print('len(multi_sparse.sparse_dgca):', len(multi_sparse.sparse_dgca)) + print('len(multi.dgca):', len(multi.dgca)) + for i in range(len(adjacency_matrices)): assert np.all(multi_sparse.sparse_dgca[i].sparse_graph_conv.weight.detach().numpy() == multi.dgca[i].graph_conv.weight.detach().numpy()) # torch.random.manual_seed(0) - latent_sparse = multi_sparse(latent_sparse) + latent_sparse = multi_sparse([latent_sparse,] * len(adjacency_matrices)) # torch.random.manual_seed(0) latent = multi(latent) -- 2.26.2 From 2e5e1d69acd53d4aaf282cfcd1e8f0df112d5b19 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 22 May 2020 14:24:00 +0200 Subject: [PATCH 022/227] Add support for multiple input_dim in MultiDGCA. --- src/decagon_pytorch/convolve.py | 24 ++++++++++++++++++++---- tests/decagon_pytorch/test_convolve.py | 6 +++--- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/decagon_pytorch/convolve.py b/src/decagon_pytorch/convolve.py index 4ff9ee8..10475c6 100644 --- a/src/decagon_pytorch/convolve.py +++ b/src/decagon_pytorch/convolve.py @@ -228,6 +228,8 @@ class SparseMultiDGCA(torch.nn.Module): self.sparse_dgca.append(SparseDropoutGraphConvActivation(input_dim, self.output_dim, adj_mat, self.keep_prob, self.activation)) def forward(self, x: List[torch.Tensor]) -> torch.Tensor: + if not isinstance(x, list): + raise ValueError('x must be a list of tensors') out = torch.zeros(len(x[0]), self.output_dim, dtype=x[0].dtype) for i, f in enumerate(self.sparse_dgca): out += f(x[i]) @@ -273,12 +275,26 @@ class MultiDGCA(torch.nn.Module): activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, **kwargs) -> None: super().__init__(**kwargs) + self.input_dim = input_dim self.output_dim = output_dim - self.dgca = [ DropoutGraphConvActivation(input_dim, output_dim, adj_mat, keep_prob, activation) for adj_mat in adjacency_matrices ] + self.adjacency_matrices = adjacency_matrices + self.keep_prob = keep_prob + self.activation = activation + self.dgca = None + self.build() + + def build(self): + if len(self.input_dim) != len(self.adjacency_matrices): + raise ValueError('input_dim must have the same length as adjacency_matrices') + self.dgca = [] + for input_dim, adj_mat in zip(self.input_dim, self.adjacency_matrices): + self.dgca.append(DropoutGraphConvActivation(input_dim, self.output_dim, adj_mat, self.keep_prob, self.activation)) def forward(self, x: List[torch.Tensor]) -> List[torch.Tensor]: - out = torch.zeros(len(x), self.output_dim, dtype=x.dtype) - for f in self.dgca: - out += f(x) + if not isinstance(x, list): + raise ValueError('x must be a list of tensors') + out = torch.zeros(len(x[0]), self.output_dim, dtype=x[0].dtype) + for i, f in enumerate(self.dgca): + out += f(x[i]) out = torch.nn.functional.normalize(out, p=2, dim=1) return out diff --git a/tests/decagon_pytorch/test_convolve.py b/tests/decagon_pytorch/test_convolve.py index 71e1ece..b529bce 100644 --- a/tests/decagon_pytorch/test_convolve.py +++ b/tests/decagon_pytorch/test_convolve.py @@ -236,10 +236,10 @@ def test_multi_dgca(): assert np.all(adjacency_matrices[i].numpy() == adjacency_matrices_sparse[i].to_dense().numpy()) torch.random.manual_seed(0) - multi_sparse = decagon_pytorch.convolve.SparseMultiDGCA([10,]*len(adjacency_matrices), 10, adjacency_matrices_sparse, keep_prob=keep_prob) + multi_sparse = decagon_pytorch.convolve.SparseMultiDGCA([10,] * len(adjacency_matrices), 10, adjacency_matrices_sparse, keep_prob=keep_prob) torch.random.manual_seed(0) - multi = decagon_pytorch.convolve.MultiDGCA(10, 10, adjacency_matrices, keep_prob=keep_prob) + multi = decagon_pytorch.convolve.MultiDGCA([10,] * len(adjacency_matrices), 10, adjacency_matrices, keep_prob=keep_prob) print('len(adjacency_matrices):', len(adjacency_matrices)) print('len(multi_sparse.sparse_dgca):', len(multi_sparse.sparse_dgca)) @@ -251,6 +251,6 @@ def test_multi_dgca(): # torch.random.manual_seed(0) latent_sparse = multi_sparse([latent_sparse,] * len(adjacency_matrices)) # torch.random.manual_seed(0) - latent = multi(latent) + latent = multi([latent,] * len(adjacency_matrices)) assert np.all(latent_sparse.detach().numpy() == latent.detach().numpy()) -- 2.26.2 From b1245e1a734a30511341b712e20d8ec6d99b74b5 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 25 May 2020 17:08:31 +0200 Subject: [PATCH 023/227] Add a diagram. --- docs/decagon-diagram.svg | 1112 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 1112 insertions(+) create mode 100644 docs/decagon-diagram.svg diff --git a/docs/decagon-diagram.svg b/docs/decagon-diagram.svg new file mode 100644 index 0000000..ac8bfb2 --- /dev/null +++ b/docs/decagon-diagram.svg @@ -0,0 +1,1112 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + Number of drugs Number of targets X + Number of drugs X Number of targets Number of drugs + + + + Number of targets X + Number of drugs 64 X 64 + + Target - Target Interactions (Adjacency Matrix) Drug - Target Relationship (Adjacency Matrix) Drug - Drug Side Effects (Adjacency Matrix) + + + + + + + + + L2norm + + L2norm + + + + + + + + + + + + + + + + + + + L2norm + + + + + + L2norm + + + + + + + + + + + + + + Number of targets X + Number of drugs 32 X 32 + + Target - Target Interactions (Adjacency Matrix) Drug - Target Relationship (Adjacency Matrix) Drug - Drug Side Effects (Adjacency Matrix) + + + + + + + + + L2norm + + L2norm + + + + + + + + + + + + + + + + + + + L2norm + + + + + + L2norm + + + + + + + + + -- 2.26.2 From c45e0fa9f11e49260d846579717ed1acd9d18a4a Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 25 May 2020 17:46:03 +0200 Subject: [PATCH 024/227] Make InputLayer support variable dimensionality representations. --- src/decagon_pytorch/layer.py | 8 ++++++-- tests/decagon_pytorch/test_layer.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/decagon_pytorch/layer.py b/src/decagon_pytorch/layer.py index a75c005..f21c182 100644 --- a/src/decagon_pytorch/layer.py +++ b/src/decagon_pytorch/layer.py @@ -25,9 +25,13 @@ from .convolve import SparseMultiDGCA class InputLayer(torch.nn.Module): - def __init__(self, data, dimensionality=32, **kwargs): + def __init__(self, data, dimensionality=None, **kwargs): super().__init__(**kwargs) self.data = data + dimensionality = dimensionality or \ + list(map(lambda a: a.count, data.node_types)) + if not isinstance(dimensionality, list): + dimensionality = [dimensionality,] * len(self.data.node_types) self.dimensionality = dimensionality self.node_reps = None self.build() @@ -35,7 +39,7 @@ class InputLayer(torch.nn.Module): def build(self): self.node_reps = [] for i, nt in enumerate(self.data.node_types): - reps = torch.rand(nt.count, self.dimensionality) + reps = torch.rand(nt.count, self.dimensionality[i]) reps = torch.nn.Parameter(reps) self.register_parameter('node_reps[%d]' % i, reps) self.node_reps.append(reps) diff --git a/tests/decagon_pytorch/test_layer.py b/tests/decagon_pytorch/test_layer.py index 7e2115c..c171e1c 100644 --- a/tests/decagon_pytorch/test_layer.py +++ b/tests/decagon_pytorch/test_layer.py @@ -1,4 +1,5 @@ -from decagon_pytorch.layer import InputLayer +from decagon_pytorch.layer import InputLayer, \ + DecagonLayer from decagon_pytorch.data import Data import torch import pytest @@ -16,11 +17,28 @@ def _some_data(): return d +def _some_data_with_interactions(): + d = Data() + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + d.add_relation_type('Target', 1, 0, + torch.rand((100, 1000), dtype=torch.float32).round()) + d.add_relation_type('Interaction', 0, 0, + torch.rand((1000, 1000), dtype=torch.float32).round()) + d.add_relation_type('Side Effect: Nausea', 1, 1, + torch.rand((100, 100), dtype=torch.float32).round()) + d.add_relation_type('Side Effect: Infertility', 1, 1, + torch.rand((100, 100), dtype=torch.float32).round()) + d.add_relation_type('Side Effect: Death', 1, 1, + torch.rand((100, 100), dtype=torch.float32).round()) + return d + + def test_input_layer_01(): d = _some_data() for dimensionality in [32, 64, 128]: layer = InputLayer(d, dimensionality) - assert layer.dimensionality == dimensionality + assert layer.dimensionality[0] == dimensionality assert len(layer.node_reps) == 2 assert layer.node_reps[0].shape == (1000, dimensionality) assert layer.node_reps[1].shape == (100, dimensionality) @@ -50,3 +68,10 @@ def test_input_layer_03(): # assert layer.device.type == 'cuda:0' assert layer.node_reps[0].device == device assert layer.node_reps[1].device == device + + +@pytest.mark.skip() +def test_decagon_layer_01(): + d = _some_data_with_interactions() + in_layer = InputLayer(d) + d_layer = DecagonLayer(in_layer, output_dim=32) -- 2.26.2 From 2cdc76fac7a467e3726f6f20d9f3a9b6808f1535 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 26 May 2020 21:17:26 +0200 Subject: [PATCH 025/227] Prepare to make Decagon layer work. --- src/decagon_pytorch/layer.py | 39 +++++++++++++++++------------ tests/decagon_pytorch/test_layer.py | 10 ++++---- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/src/decagon_pytorch/layer.py b/src/decagon_pytorch/layer.py index f21c182..d48de45 100644 --- a/src/decagon_pytorch/layer.py +++ b/src/decagon_pytorch/layer.py @@ -22,51 +22,58 @@ import torch from .convolve import SparseMultiDGCA +from .data import Data +from typing import List, Union -class InputLayer(torch.nn.Module): - def __init__(self, data, dimensionality=None, **kwargs): +class Layer(torch.nn.Module): + def __init__(self, output_dim: Union[int, List[int]], **kwargs) -> None: super().__init__(**kwargs) - self.data = data - dimensionality = dimensionality or \ + self.output_dim = output_dim + + +class InputLayer(Layer): + def __init__(self, data: Data, output_dim: Union[int, List[int]]= None, **kwargs) -> None: + output_dim = output_dim or \ list(map(lambda a: a.count, data.node_types)) - if not isinstance(dimensionality, list): - dimensionality = [dimensionality,] * len(self.data.node_types) - self.dimensionality = dimensionality + if not isinstance(output_dim, list): + output_dim = [output_dim,] * len(data.node_types) + + super().__init__(output_dim, **kwargs) + self.data = data self.node_reps = None self.build() - def build(self): + def build(self) -> None: self.node_reps = [] for i, nt in enumerate(self.data.node_types): - reps = torch.rand(nt.count, self.dimensionality[i]) + reps = torch.rand(nt.count, self.output_dim[i]) reps = torch.nn.Parameter(reps) self.register_parameter('node_reps[%d]' % i, reps) self.node_reps.append(reps) - def forward(self): + def forward(self) -> List[torch.nn.Parameter]: return self.node_reps - def __repr__(self): + def __repr__(self) -> str: s = '' - s += 'GNN input layer with dimensionality: %d\n' % self.dimensionality + s += 'GNN input layer with output_dim: %s\n' % self.output_dim s += ' # of node types: %d\n' % len(self.data.node_types) for nt in self.data.node_types: s += ' - %s (%d)\n' % (nt.name, nt.count) return s.strip() -class DecagonLayer(torch.nn.Module): - def __init__(self, data, +class DecagonLayer(Layer): + def __init__(self, data: Data, input_dim, output_dim, keep_prob=1., rel_activation=lambda x: x, layer_activation=torch.nn.functional.relu, **kwargs): - super().__init__(**kwargs) + super().__init__(output_dim, **kwargs) self.data = data self.input_dim = input_dim - self.output_dim = output_dim self.keep_prob = keep_prob self.rel_activation = rel_activation self.layer_activation = layer_activation diff --git a/tests/decagon_pytorch/test_layer.py b/tests/decagon_pytorch/test_layer.py index c171e1c..1497fe8 100644 --- a/tests/decagon_pytorch/test_layer.py +++ b/tests/decagon_pytorch/test_layer.py @@ -36,12 +36,12 @@ def _some_data_with_interactions(): def test_input_layer_01(): d = _some_data() - for dimensionality in [32, 64, 128]: - layer = InputLayer(d, dimensionality) - assert layer.dimensionality[0] == dimensionality + for output_dim in [32, 64, 128]: + layer = InputLayer(d, output_dim) + assert layer.output_dim[0] == output_dim assert len(layer.node_reps) == 2 - assert layer.node_reps[0].shape == (1000, dimensionality) - assert layer.node_reps[1].shape == (100, dimensionality) + assert layer.node_reps[0].shape == (1000, output_dim) + assert layer.node_reps[1].shape == (100, output_dim) assert layer.data == d -- 2.26.2 From 8c3e94963777a72d94773b8cd914155841e68342 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 26 May 2020 21:36:18 +0200 Subject: [PATCH 026/227] Baby steps. --- src/decagon_pytorch/data.py | 12 ++++++++++++ src/decagon_pytorch/layer.py | 21 ++++++++++++++------- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/decagon_pytorch/data.py b/src/decagon_pytorch/data.py index 0891c8b..852433f 100644 --- a/src/decagon_pytorch/data.py +++ b/src/decagon_pytorch/data.py @@ -17,6 +17,18 @@ class RelationType(object): self.node_type_column = node_type_column self.adjacency_matrix = adjacency_matrix + def get_adjacency_matrix(node_type_row, node_type_column): + if self.node_type_row == node_type_row and \ + self.node_type_column = node_type_column: + return self.adjacency_matrix + + elif self.node_type_row == node_type_column and \ + self.node_type_column == node_type_row: + return self.adjacency_matrix.transpose(0, 1) + + else: + raise ValueError('Specified row/column types do not correspond to this relation') + class Data(object): def __init__(self): diff --git a/src/decagon_pytorch/layer.py b/src/decagon_pytorch/layer.py index d48de45..bb95dae 100644 --- a/src/decagon_pytorch/layer.py +++ b/src/decagon_pytorch/layer.py @@ -23,7 +23,9 @@ import torch from .convolve import SparseMultiDGCA from .data import Data -from typing import List, Union +from typing import List, \ + Union, \ + Callable class Layer(torch.nn.Module): @@ -65,15 +67,20 @@ class InputLayer(Layer): class DecagonLayer(Layer): - def __init__(self, data: Data, - input_dim, output_dim, - keep_prob=1., - rel_activation=lambda x: x, - layer_activation=torch.nn.functional.relu, + def __init__(self, + data: Data, + previous_layer: Layer, + output_dim: Union[int, List[int]], + keep_prob: float = 1., + rel_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, + layer_activation: Callable[[torch.Tensor], torch.Tensor] = torch.nn.functional.relu, **kwargs): + if not isinstance(output_dim, list): + output_dim = [ output_dim ] * len(data.node_types) super().__init__(output_dim, **kwargs) self.data = data - self.input_dim = input_dim + self.previous_layer = previous_layer + self.input_dim = previous_layer.output_dim self.keep_prob = keep_prob self.rel_activation = rel_activation self.layer_activation = layer_activation -- 2.26.2 From d56bbf745f25965a592635598f99735e36ec3849 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 27 May 2020 18:03:08 +0200 Subject: [PATCH 027/227] Better way to compute DecagonLayer. --- docs/decagon-diagram.svg | 735 ++++++++++++++++++++++++++++++++++- src/decagon_pytorch/data.py | 4 +- src/decagon_pytorch/layer.py | 59 ++- 3 files changed, 769 insertions(+), 29 deletions(-) diff --git a/docs/decagon-diagram.svg b/docs/decagon-diagram.svg index ac8bfb2..8f8b73a 100644 --- a/docs/decagon-diagram.svg +++ b/docs/decagon-diagram.svg @@ -17,7 +17,227 @@ inkscape:version="0.92.3 (2405546, 2018-03-11)" sodipodi:docname="decagon-diagram.svg"> + id="defs2"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [ + [ + 0 0 11 0 00 1 0 [ + [ + 0 0 11 0 00 1 0 A + x + [ + [ + 0 0 11 0 00 1 0 x' + = + + 1 + + 2 + + 3 + + + + [ + [ + 0 0 11 0 00 1 0 A + + 1 + + 2 + + 3 + + + + [ + [ + 0 1 11 0 11 1 0 A2 + + 1 + + 2 + + 3 + + + + [ + [ + 0 1 00 0 11 0 0 A' diff --git a/src/decagon_pytorch/data.py b/src/decagon_pytorch/data.py index 852433f..381d15d 100644 --- a/src/decagon_pytorch/data.py +++ b/src/decagon_pytorch/data.py @@ -19,13 +19,13 @@ class RelationType(object): def get_adjacency_matrix(node_type_row, node_type_column): if self.node_type_row == node_type_row and \ - self.node_type_column = node_type_column: + self.node_type_column == node_type_column: return self.adjacency_matrix elif self.node_type_row == node_type_column and \ self.node_type_column == node_type_row: return self.adjacency_matrix.transpose(0, 1) - + else: raise ValueError('Specified row/column types do not correspond to this relation') diff --git a/src/decagon_pytorch/layer.py b/src/decagon_pytorch/layer.py index bb95dae..61911ff 100644 --- a/src/decagon_pytorch/layer.py +++ b/src/decagon_pytorch/layer.py @@ -21,11 +21,12 @@ import torch -from .convolve import SparseMultiDGCA +from .convolve import SparseDropoutGraphConvActivation from .data import Data from typing import List, \ Union, \ Callable +from collections import defaultdict class Layer(torch.nn.Module): @@ -89,30 +90,46 @@ class DecagonLayer(Layer): def build(self): self.convolutions = {} - for key in self.data.relation_types.keys(): + for (node_type_row, node_type_column) in self.data.relation_types.keys(): adjacency_matrices = \ - self.data.get_adjacency_matrices(*key) - self.convolutions[key] = SparseMultiDGCA(self.input_dim, + self.data.get_adjacency_matrices(node_type_row, node_type_column) + self.convolutions[node_type_row, node_type_column] = SparseMultiDGCA(self.input_dim, self.output_dim, adjacency_matrices, self.keep_prob, self.rel_activation) # for node_type_row, node_type_col in enumerate(self.data.node_ # if rt.node_type_row == i or rt.node_type_col == i: - def __call__(self, prev_layer_repr): - new_layer_repr = [] - for i, nt in enumerate(self.data.node_types): - new_repr = [] - for key in self.data.relation_types.keys(): - nt_row, nt_col = key - if nt_row != i and nt_col != i: - continue - if nt_row == i: - x = prev_layer_repr[nt_col] - else: - x = prev_layer_repr[nt_row] - conv = self.convolutions[key] - new_repr.append(conv(x)) - new_repr = sum(new_repr) - new_layer_repr.append(new_repr) - return new_layer_repr + def __call__(self): + prev_layer_repr = self.previous_layer() + next_layer_repr = defaultdict(list) + + for (nt_row, nt_col), rel in self.data.relation_types.items(): + conv = SparseDropoutGraphConvActivation(self.input_dim[nt_col], + self.output_dim[nt_row], rel.adjacency_matrix, + self.keep_prob, self.rel_activation) + next_layer_repr[nt_row].append(conv) + + conv = SparseDropoutGraphConvActivation(self.input_dim[nt_row], + self.output_dim[nt_col], rel.adjacency_matrix.transpose(0, 1), + self.keep_prob, self.rel_activation) + next_layer_repr[nt_col].append(conv) + + next_layer_repr = list(map(sum, next_layer_repr)) + return next_layer_repr + + + #for i, nt in enumerate(self.data.node_types): + # new_repr = [] + # for nt_row, nt_col in self.data.relation_types.keys(): + # if nt_row != i and nt_col != i: + # continue + # if nt_row == i: + # x = prev_layer_repr[nt_col] + # else: + # x = prev_layer_repr[nt_row] + # conv = self.convolutions[key] + # new_repr.append(conv(x)) + # new_repr = sum(new_repr) + # new_layer_repr.append(new_repr) + # return new_layer_repr -- 2.26.2 From 60d8a43c12ece902984496ff82e2835d9c16a130 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 27 May 2020 18:11:32 +0200 Subject: [PATCH 028/227] Fix the DecagonLayer logic. --- src/decagon_pytorch/layer.py | 43 +++++++++--------------------------- 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/src/decagon_pytorch/layer.py b/src/decagon_pytorch/layer.py index 61911ff..76f2721 100644 --- a/src/decagon_pytorch/layer.py +++ b/src/decagon_pytorch/layer.py @@ -85,51 +85,28 @@ class DecagonLayer(Layer): self.keep_prob = keep_prob self.rel_activation = rel_activation self.layer_activation = layer_activation - self.convolutions = None + self.next_layer_repr = None self.build() def build(self): - self.convolutions = {} - for (node_type_row, node_type_column) in self.data.relation_types.keys(): - adjacency_matrices = \ - self.data.get_adjacency_matrices(node_type_row, node_type_column) - self.convolutions[node_type_row, node_type_column] = SparseMultiDGCA(self.input_dim, - self.output_dim, adjacency_matrices, - self.keep_prob, self.rel_activation) - - # for node_type_row, node_type_col in enumerate(self.data.node_ - # if rt.node_type_row == i or rt.node_type_col == i: - - def __call__(self): - prev_layer_repr = self.previous_layer() - next_layer_repr = defaultdict(list) + self.next_layer_repr = defaultdict(list) for (nt_row, nt_col), rel in self.data.relation_types.items(): conv = SparseDropoutGraphConvActivation(self.input_dim[nt_col], self.output_dim[nt_row], rel.adjacency_matrix, self.keep_prob, self.rel_activation) - next_layer_repr[nt_row].append(conv) + self.next_layer_repr[nt_row].append((conv, nt_col)) conv = SparseDropoutGraphConvActivation(self.input_dim[nt_row], self.output_dim[nt_col], rel.adjacency_matrix.transpose(0, 1), self.keep_prob, self.rel_activation) - next_layer_repr[nt_col].append(conv) + self.next_layer_repr[nt_col].append((conv, nt_row)) + def __call__(self): + prev_layer_repr = self.previous_layer() + next_layer_repr = self.next_layer_repr + for i in range(len(self.data.node_types)): + next_layer_repr[i] = map(lambda conv, neighbor_type: \ + conv(prev_layer_repr[neighbor_type]), next_layer_repr[i]) next_layer_repr = list(map(sum, next_layer_repr)) return next_layer_repr - - - #for i, nt in enumerate(self.data.node_types): - # new_repr = [] - # for nt_row, nt_col in self.data.relation_types.keys(): - # if nt_row != i and nt_col != i: - # continue - # if nt_row == i: - # x = prev_layer_repr[nt_col] - # else: - # x = prev_layer_repr[nt_row] - # conv = self.convolutions[key] - # new_repr.append(conv(x)) - # new_repr = sum(new_repr) - # new_layer_repr.append(new_repr) - # return new_layer_repr -- 2.26.2 From 2188dc7ae240f9c35af07f51969facc8b6170ad3 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 27 May 2020 18:17:15 +0200 Subject: [PATCH 029/227] Basic constructor test for DecagonLayer passes. --- src/decagon_pytorch/layer.py | 21 +++++++++++---------- tests/decagon_pytorch/test_layer.py | 3 +-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/decagon_pytorch/layer.py b/src/decagon_pytorch/layer.py index 76f2721..6cd91c6 100644 --- a/src/decagon_pytorch/layer.py +++ b/src/decagon_pytorch/layer.py @@ -91,16 +91,17 @@ class DecagonLayer(Layer): def build(self): self.next_layer_repr = defaultdict(list) - for (nt_row, nt_col), rel in self.data.relation_types.items(): - conv = SparseDropoutGraphConvActivation(self.input_dim[nt_col], - self.output_dim[nt_row], rel.adjacency_matrix, - self.keep_prob, self.rel_activation) - self.next_layer_repr[nt_row].append((conv, nt_col)) - - conv = SparseDropoutGraphConvActivation(self.input_dim[nt_row], - self.output_dim[nt_col], rel.adjacency_matrix.transpose(0, 1), - self.keep_prob, self.rel_activation) - self.next_layer_repr[nt_col].append((conv, nt_row)) + for (nt_row, nt_col), relation_types in self.data.relation_types.items(): + for rel in relation_types: + conv = SparseDropoutGraphConvActivation(self.input_dim[nt_col], + self.output_dim[nt_row], rel.adjacency_matrix, + self.keep_prob, self.rel_activation) + self.next_layer_repr[nt_row].append((conv, nt_col)) + + conv = SparseDropoutGraphConvActivation(self.input_dim[nt_row], + self.output_dim[nt_col], rel.adjacency_matrix.transpose(0, 1), + self.keep_prob, self.rel_activation) + self.next_layer_repr[nt_col].append((conv, nt_row)) def __call__(self): prev_layer_repr = self.previous_layer() diff --git a/tests/decagon_pytorch/test_layer.py b/tests/decagon_pytorch/test_layer.py index 1497fe8..873ae6b 100644 --- a/tests/decagon_pytorch/test_layer.py +++ b/tests/decagon_pytorch/test_layer.py @@ -70,8 +70,7 @@ def test_input_layer_03(): assert layer.node_reps[1].device == device -@pytest.mark.skip() def test_decagon_layer_01(): d = _some_data_with_interactions() in_layer = InputLayer(d) - d_layer = DecagonLayer(in_layer, output_dim=32) + d_layer = DecagonLayer(d, in_layer, output_dim=32) -- 2.26.2 From 69dd9a49e2c46b6df68b4629ef0f2845261cc1dd Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 27 May 2020 18:50:52 +0200 Subject: [PATCH 030/227] Dummy run of DecagonLayer seems to work, that's something. --- src/decagon_pytorch/data.py | 2 ++ src/decagon_pytorch/layer.py | 56 +++++++++++++++++++++++++---- tests/decagon_pytorch/test_layer.py | 8 +++++ 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/decagon_pytorch/data.py b/src/decagon_pytorch/data.py index 381d15d..2418371 100644 --- a/src/decagon_pytorch/data.py +++ b/src/decagon_pytorch/data.py @@ -46,6 +46,8 @@ class Data(object): if node_type_row >= n or node_type_column >= n: raise ValueError('Node type index out of bounds, add node type first') key = (node_type_row, node_type_column) + if adjacency_matrix is not None and not adjacency_matrix.is_sparse: + adjacency_matrix = adjacency_matrix.to_sparse() self.relation_types[key].append(RelationType(name, node_type_row, node_type_column, adjacency_matrix)) # _ = self.decoder_types[(node_type_row, node_type_column)] diff --git a/src/decagon_pytorch/layer.py b/src/decagon_pytorch/layer.py index 6cd91c6..b1f5d70 100644 --- a/src/decagon_pytorch/layer.py +++ b/src/decagon_pytorch/layer.py @@ -30,9 +30,13 @@ from collections import defaultdict class Layer(torch.nn.Module): - def __init__(self, output_dim: Union[int, List[int]], **kwargs) -> None: + def __init__(self, + output_dim: Union[int, List[int]], + is_sparse: bool, + **kwargs) -> None: super().__init__(**kwargs) self.output_dim = output_dim + self.is_sparse = is_sparse class InputLayer(Layer): @@ -42,7 +46,7 @@ class InputLayer(Layer): if not isinstance(output_dim, list): output_dim = [output_dim,] * len(data.node_types) - super().__init__(output_dim, **kwargs) + super().__init__(output_dim, is_sparse=False, **kwargs) self.data = data self.node_reps = None self.build() @@ -67,6 +71,34 @@ class InputLayer(Layer): return s.strip() +class OneHotInputLayer(Layer): + def __init__(self, data: Data, **kwargs) -> None: + output_dim = [ a.count for a in data.node_types ] + super().__init__(output_dim, is_sparse=True, **kwargs) + self.data = data + self.node_reps = None + self.build() + + def build(self) -> None: + self.node_reps = [] + for i, nt in enumerate(self.data.node_types): + reps = torch.eye(nt.count).to_sparse() + reps = torch.nn.Parameter(reps) + self.register_parameter('node_reps[%d]' % i, reps) + self.node_reps.append(reps) + + def forward(self) -> List[torch.nn.Parameter]: + return self.node_reps + + def __repr__(self) -> str: + s = '' + s += 'One-hot GNN input layer\n' + s += ' # of node types: %d\n' % len(self.data.node_types) + for nt in self.data.node_types: + s += ' - %s (%d)\n' % (nt.name, nt.count) + return s.strip() + + class DecagonLayer(Layer): def __init__(self, data: Data, @@ -78,7 +110,7 @@ class DecagonLayer(Layer): **kwargs): if not isinstance(output_dim, list): output_dim = [ output_dim ] * len(data.node_types) - super().__init__(output_dim, **kwargs) + super().__init__(output_dim, is_sparse=False, **kwargs) self.data = data self.previous_layer = previous_layer self.input_dim = previous_layer.output_dim @@ -98,6 +130,9 @@ class DecagonLayer(Layer): self.keep_prob, self.rel_activation) self.next_layer_repr[nt_row].append((conv, nt_col)) + if nt_row == nt_col: + continue + conv = SparseDropoutGraphConvActivation(self.input_dim[nt_row], self.output_dim[nt_col], rel.adjacency_matrix.transpose(0, 1), self.keep_prob, self.rel_activation) @@ -105,9 +140,16 @@ class DecagonLayer(Layer): def __call__(self): prev_layer_repr = self.previous_layer() - next_layer_repr = self.next_layer_repr + next_layer_repr = [None] * len(self.data.node_types) + print('next_layer_repr:', next_layer_repr) for i in range(len(self.data.node_types)): - next_layer_repr[i] = map(lambda conv, neighbor_type: \ - conv(prev_layer_repr[neighbor_type]), next_layer_repr[i]) - next_layer_repr = list(map(sum, next_layer_repr)) + next_layer_repr[i] = [ + conv(prev_layer_repr[neighbor_type]) \ + for (conv, neighbor_type) in \ + self.next_layer_repr[i] + ] + next_layer_repr[i] = sum(next_layer_repr[i]) + + print('next_layer_repr:', next_layer_repr) + # next_layer_repr = list(map(sum, next_layer_repr)) return next_layer_repr diff --git a/tests/decagon_pytorch/test_layer.py b/tests/decagon_pytorch/test_layer.py index 873ae6b..4dee8b1 100644 --- a/tests/decagon_pytorch/test_layer.py +++ b/tests/decagon_pytorch/test_layer.py @@ -1,4 +1,5 @@ from decagon_pytorch.layer import InputLayer, \ + OneHotInputLayer, \ DecagonLayer from decagon_pytorch.data import Data import torch @@ -74,3 +75,10 @@ def test_decagon_layer_01(): d = _some_data_with_interactions() in_layer = InputLayer(d) d_layer = DecagonLayer(d, in_layer, output_dim=32) + + +def test_decagon_layer_02(): + d = _some_data_with_interactions() + in_layer = OneHotInputLayer(d) + d_layer = DecagonLayer(d, in_layer, output_dim=32) + _ = d_layer() # dummy call -- 2.26.2 From 5d8c2d08c41e734cb55a1957d85653526c897e0f Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 28 May 2020 10:41:41 +0200 Subject: [PATCH 031/227] Add tests for OneHotInputLayer. --- tests/decagon_pytorch/test_layer.py | 40 +++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/decagon_pytorch/test_layer.py b/tests/decagon_pytorch/test_layer.py index 4dee8b1..16354ab 100644 --- a/tests/decagon_pytorch/test_layer.py +++ b/tests/decagon_pytorch/test_layer.py @@ -71,6 +71,42 @@ def test_input_layer_03(): assert layer.node_reps[1].device == device +def test_one_hot_input_layer_01(): + d = _some_data() + layer = OneHotInputLayer(d) + assert layer.output_dim == [1000, 100] + assert len(layer.node_reps) == 2 + assert layer.node_reps[0].shape == (1000, 1000) + assert layer.node_reps[1].shape == (100, 100) + assert layer.data == d + assert layer.is_sparse + + +def test_one_hot_input_layer_02(): + d = _some_data() + layer = OneHotInputLayer(d) + res = layer() + assert isinstance(res[0], torch.Tensor) + assert isinstance(res[1], torch.Tensor) + assert res[0].shape == (1000, 1000) + assert res[1].shape == (100, 100) + assert torch.all(res[0].to_dense() == layer.node_reps[0].to_dense()) + assert torch.all(res[1].to_dense() == layer.node_reps[1].to_dense()) + + +def test_one_hot_input_layer_03(): + if torch.cuda.device_count() == 0: + pytest.skip('No CUDA devices on this host') + d = _some_data() + layer = OneHotInputLayer(d) + device = torch.device('cuda:0') + layer = layer.to(device) + print(list(layer.parameters())) + # assert layer.device.type == 'cuda:0' + assert layer.node_reps[0].device == device + assert layer.node_reps[1].device == device + + def test_decagon_layer_01(): d = _some_data_with_interactions() in_layer = InputLayer(d) @@ -82,3 +118,7 @@ def test_decagon_layer_02(): in_layer = OneHotInputLayer(d) d_layer = DecagonLayer(d, in_layer, output_dim=32) _ = d_layer() # dummy call + + +def test_decagon_layer_03(): + pass -- 2.26.2 From 29e81f4eba52f738f01cd11882864bb9b20d7dac Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 28 May 2020 11:24:10 +0200 Subject: [PATCH 032/227] Add test for DecagonLayer. --- src/decagon_pytorch/convolve.py | 5 ++++- tests/decagon_pytorch/test_layer.py | 31 ++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/decagon_pytorch/convolve.py b/src/decagon_pytorch/convolve.py index 10475c6..9087f6d 100644 --- a/src/decagon_pytorch/convolve.py +++ b/src/decagon_pytorch/convolve.py @@ -195,9 +195,12 @@ class SparseDropoutGraphConvActivation(torch.nn.Module): activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, **kwargs) -> None: super().__init__(**kwargs) - self.sparse_graph_conv = SparseGraphConv(input_dim, output_dim, adjacency_matrix) + self.input_dim = input_dim + self.output_dim = output_dim + self.adjacency_matrix = adjacency_matrix self.keep_prob = keep_prob self.activation = activation + self.sparse_graph_conv = SparseGraphConv(input_dim, output_dim, adjacency_matrix) def forward(self, x: torch.Tensor) -> torch.Tensor: x = dropout_sparse(x, self.keep_prob) diff --git a/tests/decagon_pytorch/test_layer.py b/tests/decagon_pytorch/test_layer.py index 16354ab..cfb5063 100644 --- a/tests/decagon_pytorch/test_layer.py +++ b/tests/decagon_pytorch/test_layer.py @@ -4,6 +4,7 @@ from decagon_pytorch.layer import InputLayer, \ from decagon_pytorch.data import Data import torch import pytest +from decagon_pytorch.convolve import SparseDropoutGraphConvActivation def _some_data(): @@ -121,4 +122,32 @@ def test_decagon_layer_02(): def test_decagon_layer_03(): - pass + d = _some_data_with_interactions() + in_layer = OneHotInputLayer(d) + d_layer = DecagonLayer(d, in_layer, output_dim=32) + assert d_layer.data == d + assert d_layer.previous_layer == in_layer + assert d_layer.input_dim == [ 1000, 100 ] + assert not d_layer.is_sparse + assert d_layer.keep_prob == 1. + assert d_layer.rel_activation(0.5) == 0.5 + x = torch.tensor([-1, 0, 0.5, 1]) + assert (d_layer.layer_activation(x) == torch.nn.functional.relu(x)).all() + assert len(d_layer.next_layer_repr) == 2 + assert len(d_layer.next_layer_repr[0]) == 2 + assert len(d_layer.next_layer_repr[1]) == 4 + assert all(map(lambda a: isinstance(a[0], SparseDropoutGraphConvActivation), + d_layer.next_layer_repr[0])) + assert all(map(lambda a: isinstance(a[0], SparseDropoutGraphConvActivation), + d_layer.next_layer_repr[1])) + assert all(map(lambda a: a[0].output_dim == 32, + d_layer.next_layer_repr[0])) + assert all(map(lambda a: a[0].output_dim == 32, + d_layer.next_layer_repr[1])) + + +def test_decagon_layer_04(): + d = _some_data_with_interactions() + in_layer = OneHotInputLayer(d) + d_layer = DecagonLayer(d, in_layer, output_dim=32) + _ = d_layer() -- 2.26.2 From 99dbcdeb916b23348320a4568e43460a84c465d9 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 28 May 2020 11:59:43 +0200 Subject: [PATCH 033/227] Prefix dense versions of convolution classes with Dense. --- src/decagon_pytorch/convolve.py | 10 +++++----- tests/decagon_pytorch/test_convolve.py | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/decagon_pytorch/convolve.py b/src/decagon_pytorch/convolve.py index 9087f6d..d58c35f 100644 --- a/src/decagon_pytorch/convolve.py +++ b/src/decagon_pytorch/convolve.py @@ -240,7 +240,7 @@ class SparseMultiDGCA(torch.nn.Module): return out -class GraphConv(torch.nn.Module): +class DenseGraphConv(torch.nn.Module): def __init__(self, in_channels: int, out_channels: int, adjacency_matrix: torch.Tensor, **kwargs) -> None: super().__init__(**kwargs) @@ -255,13 +255,13 @@ class GraphConv(torch.nn.Module): return x -class DropoutGraphConvActivation(torch.nn.Module): +class DenseDropoutGraphConvActivation(torch.nn.Module): def __init__(self, input_dim: int, output_dim: int, adjacency_matrix: torch.Tensor, keep_prob: float=1., activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, **kwargs) -> None: super().__init__(**kwargs) - self.graph_conv = GraphConv(input_dim, output_dim, adjacency_matrix) + self.graph_conv = DenseGraphConv(input_dim, output_dim, adjacency_matrix) self.keep_prob = keep_prob self.activation = activation @@ -272,7 +272,7 @@ class DropoutGraphConvActivation(torch.nn.Module): return x -class MultiDGCA(torch.nn.Module): +class DenseMultiDGCA(torch.nn.Module): def __init__(self, input_dim: List[int], output_dim: int, adjacency_matrices: List[torch.Tensor], keep_prob: float=1., activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, @@ -291,7 +291,7 @@ class MultiDGCA(torch.nn.Module): raise ValueError('input_dim must have the same length as adjacency_matrices') self.dgca = [] for input_dim, adj_mat in zip(self.input_dim, self.adjacency_matrices): - self.dgca.append(DropoutGraphConvActivation(input_dim, self.output_dim, adj_mat, self.keep_prob, self.activation)) + self.dgca.append(DenseDropoutGraphConvActivation(input_dim, self.output_dim, adj_mat, self.keep_prob, self.activation)) def forward(self, x: List[torch.Tensor]) -> List[torch.Tensor]: if not isinstance(x, list): diff --git a/tests/decagon_pytorch/test_convolve.py b/tests/decagon_pytorch/test_convolve.py index b529bce..302eec2 100644 --- a/tests/decagon_pytorch/test_convolve.py +++ b/tests/decagon_pytorch/test_convolve.py @@ -41,25 +41,25 @@ def dropout_sparse_tf(x, keep_prob, num_nonzero_elems): return pre_out * (1./keep_prob) -def graph_conv_torch(): +def dense_graph_conv_torch(): torch.random.manual_seed(0) latent, adjacency_matrices = prepare_data() latent = torch.tensor(latent) adj_mat = adjacency_matrices[0] adj_mat = torch.tensor(adj_mat) - conv = decagon_pytorch.convolve.GraphConv(10, 10, + conv = decagon_pytorch.convolve.DenseGraphConv(10, 10, adj_mat) latent = conv(latent) return latent -def dropout_graph_conv_activation_torch(keep_prob=1.): +def dense_dropout_graph_conv_activation_torch(keep_prob=1.): torch.random.manual_seed(0) latent, adjacency_matrices = prepare_data() latent = torch.tensor(latent) adj_mat = adjacency_matrices[0] adj_mat = torch.tensor(adj_mat) - conv = decagon_pytorch.convolve.DropoutGraphConvActivation(10, 10, + conv = decagon_pytorch.convolve.DenseDropoutGraphConvActivation(10, 10, adj_mat, keep_prob=keep_prob) latent = conv(latent) return latent @@ -173,7 +173,7 @@ def test_sparse_multi_dgca(): def test_graph_conv(): - latent_dense = graph_conv_torch() + latent_dense = dense_graph_conv_torch() latent_sparse = sparse_graph_conv_torch() assert np.all(latent_dense.detach().numpy() == latent_sparse.detach().numpy()) @@ -206,7 +206,7 @@ def test_dropout_graph_conv_activation(): keep_prob += np.finfo(np.float32).eps print('keep_prob:', keep_prob) - latent_dense = dropout_graph_conv_activation_torch(keep_prob) + latent_dense = dense_dropout_graph_conv_activation_torch(keep_prob) latent_dense = latent_dense.detach().numpy() print('latent_dense:', latent_dense) @@ -239,7 +239,7 @@ def test_multi_dgca(): multi_sparse = decagon_pytorch.convolve.SparseMultiDGCA([10,] * len(adjacency_matrices), 10, adjacency_matrices_sparse, keep_prob=keep_prob) torch.random.manual_seed(0) - multi = decagon_pytorch.convolve.MultiDGCA([10,] * len(adjacency_matrices), 10, adjacency_matrices, keep_prob=keep_prob) + multi = decagon_pytorch.convolve.DenseMultiDGCA([10,] * len(adjacency_matrices), 10, adjacency_matrices, keep_prob=keep_prob) print('len(adjacency_matrices):', len(adjacency_matrices)) print('len(multi_sparse.sparse_dgca):', len(multi_sparse.sparse_dgca)) -- 2.26.2 From 69e8ce5081458d2c83ae5fc9f87e4b031501bbb1 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 29 May 2020 09:51:50 +0200 Subject: [PATCH 034/227] Add universal convolution units that can handle both sparse and dense inputs. --- src/decagon_pytorch/convolve.py | 43 ++++++++++++++++++++++++++ tests/decagon_pytorch/test_convolve.py | 24 ++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/src/decagon_pytorch/convolve.py b/src/decagon_pytorch/convolve.py index d58c35f..4048521 100644 --- a/src/decagon_pytorch/convolve.py +++ b/src/decagon_pytorch/convolve.py @@ -301,3 +301,46 @@ class DenseMultiDGCA(torch.nn.Module): out += f(x[i]) out = torch.nn.functional.normalize(out, p=2, dim=1) return out + + +class GraphConv(torch.nn.Module): + """Convolution layer for sparse AND dense inputs.""" + def __init__(self, in_channels: int, out_channels: int, + adjacency_matrix: torch.Tensor, **kwargs) -> None: + super().__init__(**kwargs) + self.in_channels = in_channels + self.out_channels = out_channels + self.weight = init_glorot(in_channels, out_channels) + self.adjacency_matrix = adjacency_matrix + + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = torch.sparse.mm(x, self.weight) \ + if x.is_sparse \ + else torch.mm(x, self.weight) + x = torch.sparse.mm(self.adjacency_matrix, x) \ + if self.adjacency_matrix.is_sparse \ + else torch.mm(self.adjacency_matrix, x) + return x + + +class DropoutGraphConvActivation(torch.nn.Module): + def __init__(self, input_dim: int, output_dim: int, + adjacency_matrix: torch.Tensor, keep_prob: float=1., + activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, + **kwargs) -> None: + super().__init__(**kwargs) + self.input_dim = input_dim + self.output_dim = output_dim + self.adjacency_matrix = adjacency_matrix + self.keep_prob = keep_prob + self.activation = activation + self.graph_conv = GraphConv(input_dim, output_dim, adjacency_matrix) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = dropout_sparse(x, self.keep_prob) \ + if x.is_sparse \ + else dropout(x, self.keep_prob) + x = self.graph_conv(x) + x = self.activation(x) + return x diff --git a/tests/decagon_pytorch/test_convolve.py b/tests/decagon_pytorch/test_convolve.py index 302eec2..1f8b7a0 100644 --- a/tests/decagon_pytorch/test_convolve.py +++ b/tests/decagon_pytorch/test_convolve.py @@ -199,6 +199,18 @@ def teardown_function(fun): setup_function.old_dropout +def flexible_dropout_graph_conv_activation_torch(keep_prob=1.): + torch.random.manual_seed(0) + latent, adjacency_matrices = prepare_data() + latent = torch.tensor(latent).to_sparse() + adj_mat = adjacency_matrices[0] + adj_mat = torch.tensor(adj_mat).to_sparse() + conv = decagon_pytorch.convolve.DropoutGraphConvActivation(10, 10, + adj_mat, keep_prob=keep_prob) + latent = conv(latent) + return latent + + def test_dropout_graph_conv_activation(): for i in range(11): keep_prob = i/10. @@ -214,10 +226,22 @@ def test_dropout_graph_conv_activation(): latent_sparse = latent_sparse.detach().numpy() print('latent_sparse:', latent_sparse) + latent_flex = flexible_dropout_graph_conv_activation_torch(keep_prob) + latent_flex = latent_flex.detach().numpy() + print('latent_flex:', latent_flex) + nonzero = (latent_dense != 0) & (latent_sparse != 0) assert np.all(latent_dense[nonzero] == latent_sparse[nonzero]) + nonzero = (latent_dense != 0) & (latent_flex != 0) + + assert np.all(latent_dense[nonzero] == latent_flex[nonzero]) + + nonzero = (latent_sparse != 0) & (latent_flex != 0) + + assert np.all(latent_sparse[nonzero] == latent_flex[nonzero]) + def test_multi_dgca(): keep_prob = .5 -- 2.26.2 From 02bbfc4958de5ad87fd34f1500502d06246ded7c Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 29 May 2020 10:38:08 +0200 Subject: [PATCH 035/227] Add test_decagon_layer_04(). --- src/decagon_pytorch/layer.py | 7 +++-- tests/decagon_pytorch/test_layer.py | 45 +++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/src/decagon_pytorch/layer.py b/src/decagon_pytorch/layer.py index b1f5d70..69ca74c 100644 --- a/src/decagon_pytorch/layer.py +++ b/src/decagon_pytorch/layer.py @@ -21,7 +21,7 @@ import torch -from .convolve import SparseDropoutGraphConvActivation +from .convolve import DropoutGraphConvActivation from .data import Data from typing import List, \ Union, \ @@ -125,7 +125,7 @@ class DecagonLayer(Layer): for (nt_row, nt_col), relation_types in self.data.relation_types.items(): for rel in relation_types: - conv = SparseDropoutGraphConvActivation(self.input_dim[nt_col], + conv = DropoutGraphConvActivation(self.input_dim[nt_col], self.output_dim[nt_row], rel.adjacency_matrix, self.keep_prob, self.rel_activation) self.next_layer_repr[nt_row].append((conv, nt_col)) @@ -133,7 +133,7 @@ class DecagonLayer(Layer): if nt_row == nt_col: continue - conv = SparseDropoutGraphConvActivation(self.input_dim[nt_row], + conv = DropoutGraphConvActivation(self.input_dim[nt_row], self.output_dim[nt_col], rel.adjacency_matrix.transpose(0, 1), self.keep_prob, self.rel_activation) self.next_layer_repr[nt_col].append((conv, nt_row)) @@ -149,6 +149,7 @@ class DecagonLayer(Layer): self.next_layer_repr[i] ] next_layer_repr[i] = sum(next_layer_repr[i]) + next_layer_repr[i] = torch.nn.functional.normalize(next_layer_repr[i], p=2, dim=1) print('next_layer_repr:', next_layer_repr) # next_layer_repr = list(map(sum, next_layer_repr)) diff --git a/tests/decagon_pytorch/test_layer.py b/tests/decagon_pytorch/test_layer.py index cfb5063..cc0e5b7 100644 --- a/tests/decagon_pytorch/test_layer.py +++ b/tests/decagon_pytorch/test_layer.py @@ -4,7 +4,9 @@ from decagon_pytorch.layer import InputLayer, \ from decagon_pytorch.data import Data import torch import pytest -from decagon_pytorch.convolve import SparseDropoutGraphConvActivation +from decagon_pytorch.convolve import SparseDropoutGraphConvActivation, \ + SparseMultiDGCA, \ + DropoutGraphConvActivation def _some_data(): @@ -136,9 +138,9 @@ def test_decagon_layer_03(): assert len(d_layer.next_layer_repr) == 2 assert len(d_layer.next_layer_repr[0]) == 2 assert len(d_layer.next_layer_repr[1]) == 4 - assert all(map(lambda a: isinstance(a[0], SparseDropoutGraphConvActivation), + assert all(map(lambda a: isinstance(a[0], DropoutGraphConvActivation), d_layer.next_layer_repr[0])) - assert all(map(lambda a: isinstance(a[0], SparseDropoutGraphConvActivation), + assert all(map(lambda a: isinstance(a[0], DropoutGraphConvActivation), d_layer.next_layer_repr[1])) assert all(map(lambda a: a[0].output_dim == 32, d_layer.next_layer_repr[0])) @@ -147,7 +149,38 @@ def test_decagon_layer_03(): def test_decagon_layer_04(): - d = _some_data_with_interactions() + # check if it is equivalent to MultiDGCA, as it should be + + d = Data() + d.add_node_type('Dummy', 100) + d.add_relation_type('Dummy Relation', 0, 0, + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + in_layer = OneHotInputLayer(d) - d_layer = DecagonLayer(d, in_layer, output_dim=32) - _ = d_layer() + + multi_dgca = SparseMultiDGCA([10], 32, + [r.adjacency_matrix for r in d.relation_types[0, 0]], + keep_prob=1., activation=lambda x: x) + + d_layer = DecagonLayer(d, in_layer, output_dim=32, + keep_prob=1., rel_activation=lambda x: x, + layer_activation=lambda x: x) + + assert isinstance(d_layer.next_layer_repr[0][0][0], + DropoutGraphConvActivation) + + weight = d_layer.next_layer_repr[0][0][0].graph_conv.weight + assert isinstance(weight, torch.Tensor) + + assert len(multi_dgca.sparse_dgca) == 1 + assert isinstance(multi_dgca.sparse_dgca[0], SparseDropoutGraphConvActivation) + + multi_dgca.sparse_dgca[0].sparse_graph_conv.weight = weight + + out_d_layer = d_layer() + out_multi_dgca = multi_dgca(in_layer()) + + assert isinstance(out_d_layer, list) + assert len(out_d_layer) == 1 + + assert torch.all(out_d_layer[0] == out_multi_dgca) -- 2.26.2 From c74555fef51aec638d50b198d002cbd1bdc84e01 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 29 May 2020 10:46:57 +0200 Subject: [PATCH 036/227] Refactor convolve into 3 separate modules. --- src/decagon_pytorch/convolve.py | 346 ---------------------- src/decagon_pytorch/convolve/__init__.py | 168 +++++++++++ src/decagon_pytorch/convolve/dense.py | 73 +++++ src/decagon_pytorch/convolve/sparse.py | 78 +++++ src/decagon_pytorch/convolve/universal.py | 85 ++++++ 5 files changed, 404 insertions(+), 346 deletions(-) delete mode 100644 src/decagon_pytorch/convolve.py create mode 100644 src/decagon_pytorch/convolve/__init__.py create mode 100644 src/decagon_pytorch/convolve/dense.py create mode 100644 src/decagon_pytorch/convolve/sparse.py create mode 100644 src/decagon_pytorch/convolve/universal.py diff --git a/src/decagon_pytorch/convolve.py b/src/decagon_pytorch/convolve.py deleted file mode 100644 index 4048521..0000000 --- a/src/decagon_pytorch/convolve.py +++ /dev/null @@ -1,346 +0,0 @@ -# -# Copyright (C) Stanislaw Adaszewski, 2020 -# License: GPLv3 -# - -""" -This module implements the basic convolutional blocks of Decagon. -Just as a quick reminder, the basic convolution formula here is: - -y = A * (x * W) - -where: - -W is a weight matrix -A is an adjacency matrix -x is a matrix of latent representations of a particular type of neighbors. - -As we have x here twice, a trick is obviously necessary for this to work. -A must be previously normalized with: - -c_{r}^{ij} = 1/sqrt(|N_{r}^{i}| |N_{r}^{j}|) - -or - -c_{r}^{i} = 1/|N_{r}^{i}| - -Let's work through this step by step to convince ourselves that the -formula is correct. - -x = [ - [0, 1, 0, 1], - [1, 1, 1, 0], - [0, 0, 0, 1] -] - -W = [ - [0, 1], - [1, 0], - [0.5, 0.5], - [0.25, 0.75] -] - -A = [ - [0, 1, 0], - [1, 0, 1], - [0, 1, 0] -] - -so the graph looks like this: - -(0) -- (1) -- (2) - -and therefore the representations in the next layer should be: - -h_{0}^{k+1} = c_{r}^{0,1} * h_{1}^{k} * W + c_{r}^{0} * h_{0}^{k} -h_{1}^{k+1} = c_{r}^{0,1} * h_{0}^{k} * W + c_{r}^{2,1} * h_{2}^{k} + - c_{r}^{1} * h_{1}^{k} -h_{2}^{k+1} = c_{r}^{2,1} * h_{1}^{k} * W + c_{r}^{2} * h_{2}^{k} - -In actual Decagon code we can see that that latter part propagating directly -the old representation is gone. I will try to do the same for now. - -So we have to only take care of: - -h_{0}^{k+1} = c_{r}^{0,1} * h_{1}^{k} * W -h_{1}^{k+1} = c_{r}^{0,1} * h_{0}^{k} * W + c_{r}^{2,1} * h_{2}^{k} -h_{2}^{k+1} = c_{r}^{2,1} * h_{1}^{k} * W - -If A is square the Decagon's EdgeMinibatchIterator preprocesses it as follows: - -A = A + eye(len(A)) -rowsum = A.sum(1) -deg_mat_inv_sqrt = diags(power(rowsum, -0.5)) -A = dot(A, deg_mat_inv_sqrt) -A = A.transpose() -A = A.dot(deg_mat_inv_sqrt) - -Let's see what gives in our case: - -A = A + eye(len(A)) - -[ - [1, 1, 0], - [1, 1, 1], - [0, 1, 1] -] - -rowsum = A.sum(1) - -[2, 3, 2] - -deg_mat_inv_sqrt = diags(power(rowsum, -0.5)) - -[ - [1./sqrt(2), 0, 0], - [0, 1./sqrt(3), 0], - [0, 0, 1./sqrt(2)] -] - -A = dot(A, deg_mat_inv_sqrt) - -[ - [ 1/sqrt(2), 1/sqrt(3), 0 ], - [ 1/sqrt(2), 1/sqrt(3), 1/sqrt(2) ], - [ 0, 1/sqrt(3), 1/sqrt(2) ] -] - -A = A.transpose() - -[ - [ 1/sqrt(2), 1/sqrt(2), 0 ], - [ 1/sqrt(3), 1/sqrt(3), 1/sqrt(3) ], - [ 0, 1/sqrt(2), 1/sqrt(2) ] -] - -A = A.dot(deg_mat_inv_sqrt) - -[ - [ 1/sqrt(2) * 1/sqrt(2), 1/sqrt(2) * 1/sqrt(3), 0 ], - [ 1/sqrt(3) * 1/sqrt(2), 1/sqrt(3) * 1/sqrt(3), 1/sqrt(3) * 1/sqrt(2) ], - [ 0, 1/sqrt(2) * 1/sqrt(3), 1/sqrt(2) * 1/sqrt(2) ], -] - -thus: - -[ - [0.5 , 0.40824829, 0. ], - [0.40824829, 0.33333333, 0.40824829], - [0. , 0.40824829, 0.5 ] -] - -This checks out with the 1/sqrt(|N_{r}^{i}| |N_{r}^{j}|) formula. - -Then, we get back to the main calculation: - -y = x * W -y = A * y - -y = x * W - -[ - [ 1.25, 0.75 ], - [ 1.5 , 1.5 ], - [ 0.25, 0.75 ] -] - -y = A * y - -[ - 0.5 * [ 1.25, 0.75 ] + 0.40824829 * [ 1.5, 1.5 ], - 0.40824829 * [ 1.25, 0.75 ] + 0.33333333 * [ 1.5, 1.5 ] + 0.40824829 * [ 0.25, 0.75 ], - 0.40824829 * [ 1.5, 1.5 ] + 0.5 * [ 0.25, 0.75 ] -] - -that is: - -[ - [1.23737243, 0.98737244], - [1.11237243, 1.11237243], - [0.73737244, 0.98737244] -]. - -All checks out nicely, good. - -""" - - -import torch -from .dropout import dropout_sparse, \ - dropout -from .weights import init_glorot -from typing import List, Callable - - -class SparseGraphConv(torch.nn.Module): - """Convolution layer for sparse inputs.""" - def __init__(self, in_channels: int, out_channels: int, - adjacency_matrix: torch.Tensor, **kwargs) -> None: - super().__init__(**kwargs) - self.in_channels = in_channels - self.out_channels = out_channels - self.weight = init_glorot(in_channels, out_channels) - self.adjacency_matrix = adjacency_matrix - - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x = torch.sparse.mm(x, self.weight) - x = torch.sparse.mm(self.adjacency_matrix, x) - return x - - -class SparseDropoutGraphConvActivation(torch.nn.Module): - def __init__(self, input_dim: int, output_dim: int, - adjacency_matrix: torch.Tensor, keep_prob: float=1., - activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, - **kwargs) -> None: - super().__init__(**kwargs) - self.input_dim = input_dim - self.output_dim = output_dim - self.adjacency_matrix = adjacency_matrix - self.keep_prob = keep_prob - self.activation = activation - self.sparse_graph_conv = SparseGraphConv(input_dim, output_dim, adjacency_matrix) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x = dropout_sparse(x, self.keep_prob) - x = self.sparse_graph_conv(x) - x = self.activation(x) - return x - - -class SparseMultiDGCA(torch.nn.Module): - def __init__(self, input_dim: List[int], output_dim: int, - adjacency_matrices: List[torch.Tensor], keep_prob: float=1., - activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, - **kwargs) -> None: - super().__init__(**kwargs) - self.input_dim = input_dim - self.output_dim = output_dim - self.adjacency_matrices = adjacency_matrices - self.keep_prob = keep_prob - self.activation = activation - self.sparse_dgca = None - self.build() - - def build(self): - if len(self.input_dim) != len(self.adjacency_matrices): - raise ValueError('input_dim must have the same length as adjacency_matrices') - self.sparse_dgca = [] - for input_dim, adj_mat in zip(self.input_dim, self.adjacency_matrices): - self.sparse_dgca.append(SparseDropoutGraphConvActivation(input_dim, self.output_dim, adj_mat, self.keep_prob, self.activation)) - - def forward(self, x: List[torch.Tensor]) -> torch.Tensor: - if not isinstance(x, list): - raise ValueError('x must be a list of tensors') - out = torch.zeros(len(x[0]), self.output_dim, dtype=x[0].dtype) - for i, f in enumerate(self.sparse_dgca): - out += f(x[i]) - out = torch.nn.functional.normalize(out, p=2, dim=1) - return out - - -class DenseGraphConv(torch.nn.Module): - def __init__(self, in_channels: int, out_channels: int, - adjacency_matrix: torch.Tensor, **kwargs) -> None: - super().__init__(**kwargs) - self.in_channels = in_channels - self.out_channels = out_channels - self.weight = init_glorot(in_channels, out_channels) - self.adjacency_matrix = adjacency_matrix - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x = torch.mm(x, self.weight) - x = torch.mm(self.adjacency_matrix, x) - return x - - -class DenseDropoutGraphConvActivation(torch.nn.Module): - def __init__(self, input_dim: int, output_dim: int, - adjacency_matrix: torch.Tensor, keep_prob: float=1., - activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, - **kwargs) -> None: - super().__init__(**kwargs) - self.graph_conv = DenseGraphConv(input_dim, output_dim, adjacency_matrix) - self.keep_prob = keep_prob - self.activation = activation - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x = dropout(x, keep_prob=self.keep_prob) - x = self.graph_conv(x) - x = self.activation(x) - return x - - -class DenseMultiDGCA(torch.nn.Module): - def __init__(self, input_dim: List[int], output_dim: int, - adjacency_matrices: List[torch.Tensor], keep_prob: float=1., - activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, - **kwargs) -> None: - super().__init__(**kwargs) - self.input_dim = input_dim - self.output_dim = output_dim - self.adjacency_matrices = adjacency_matrices - self.keep_prob = keep_prob - self.activation = activation - self.dgca = None - self.build() - - def build(self): - if len(self.input_dim) != len(self.adjacency_matrices): - raise ValueError('input_dim must have the same length as adjacency_matrices') - self.dgca = [] - for input_dim, adj_mat in zip(self.input_dim, self.adjacency_matrices): - self.dgca.append(DenseDropoutGraphConvActivation(input_dim, self.output_dim, adj_mat, self.keep_prob, self.activation)) - - def forward(self, x: List[torch.Tensor]) -> List[torch.Tensor]: - if not isinstance(x, list): - raise ValueError('x must be a list of tensors') - out = torch.zeros(len(x[0]), self.output_dim, dtype=x[0].dtype) - for i, f in enumerate(self.dgca): - out += f(x[i]) - out = torch.nn.functional.normalize(out, p=2, dim=1) - return out - - -class GraphConv(torch.nn.Module): - """Convolution layer for sparse AND dense inputs.""" - def __init__(self, in_channels: int, out_channels: int, - adjacency_matrix: torch.Tensor, **kwargs) -> None: - super().__init__(**kwargs) - self.in_channels = in_channels - self.out_channels = out_channels - self.weight = init_glorot(in_channels, out_channels) - self.adjacency_matrix = adjacency_matrix - - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x = torch.sparse.mm(x, self.weight) \ - if x.is_sparse \ - else torch.mm(x, self.weight) - x = torch.sparse.mm(self.adjacency_matrix, x) \ - if self.adjacency_matrix.is_sparse \ - else torch.mm(self.adjacency_matrix, x) - return x - - -class DropoutGraphConvActivation(torch.nn.Module): - def __init__(self, input_dim: int, output_dim: int, - adjacency_matrix: torch.Tensor, keep_prob: float=1., - activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, - **kwargs) -> None: - super().__init__(**kwargs) - self.input_dim = input_dim - self.output_dim = output_dim - self.adjacency_matrix = adjacency_matrix - self.keep_prob = keep_prob - self.activation = activation - self.graph_conv = GraphConv(input_dim, output_dim, adjacency_matrix) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x = dropout_sparse(x, self.keep_prob) \ - if x.is_sparse \ - else dropout(x, self.keep_prob) - x = self.graph_conv(x) - x = self.activation(x) - return x diff --git a/src/decagon_pytorch/convolve/__init__.py b/src/decagon_pytorch/convolve/__init__.py new file mode 100644 index 0000000..f3c2e40 --- /dev/null +++ b/src/decagon_pytorch/convolve/__init__.py @@ -0,0 +1,168 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + +""" +This module implements the basic convolutional blocks of Decagon. +Just as a quick reminder, the basic convolution formula here is: + +y = A * (x * W) + +where: + +W is a weight matrix +A is an adjacency matrix +x is a matrix of latent representations of a particular type of neighbors. + +As we have x here twice, a trick is obviously necessary for this to work. +A must be previously normalized with: + +c_{r}^{ij} = 1/sqrt(|N_{r}^{i}| |N_{r}^{j}|) + +or + +c_{r}^{i} = 1/|N_{r}^{i}| + +Let's work through this step by step to convince ourselves that the +formula is correct. + +x = [ + [0, 1, 0, 1], + [1, 1, 1, 0], + [0, 0, 0, 1] +] + +W = [ + [0, 1], + [1, 0], + [0.5, 0.5], + [0.25, 0.75] +] + +A = [ + [0, 1, 0], + [1, 0, 1], + [0, 1, 0] +] + +so the graph looks like this: + +(0) -- (1) -- (2) + +and therefore the representations in the next layer should be: + +h_{0}^{k+1} = c_{r}^{0,1} * h_{1}^{k} * W + c_{r}^{0} * h_{0}^{k} +h_{1}^{k+1} = c_{r}^{0,1} * h_{0}^{k} * W + c_{r}^{2,1} * h_{2}^{k} + + c_{r}^{1} * h_{1}^{k} +h_{2}^{k+1} = c_{r}^{2,1} * h_{1}^{k} * W + c_{r}^{2} * h_{2}^{k} + +In actual Decagon code we can see that that latter part propagating directly +the old representation is gone. I will try to do the same for now. + +So we have to only take care of: + +h_{0}^{k+1} = c_{r}^{0,1} * h_{1}^{k} * W +h_{1}^{k+1} = c_{r}^{0,1} * h_{0}^{k} * W + c_{r}^{2,1} * h_{2}^{k} +h_{2}^{k+1} = c_{r}^{2,1} * h_{1}^{k} * W + +If A is square the Decagon's EdgeMinibatchIterator preprocesses it as follows: + +A = A + eye(len(A)) +rowsum = A.sum(1) +deg_mat_inv_sqrt = diags(power(rowsum, -0.5)) +A = dot(A, deg_mat_inv_sqrt) +A = A.transpose() +A = A.dot(deg_mat_inv_sqrt) + +Let's see what gives in our case: + +A = A + eye(len(A)) + +[ + [1, 1, 0], + [1, 1, 1], + [0, 1, 1] +] + +rowsum = A.sum(1) + +[2, 3, 2] + +deg_mat_inv_sqrt = diags(power(rowsum, -0.5)) + +[ + [1./sqrt(2), 0, 0], + [0, 1./sqrt(3), 0], + [0, 0, 1./sqrt(2)] +] + +A = dot(A, deg_mat_inv_sqrt) + +[ + [ 1/sqrt(2), 1/sqrt(3), 0 ], + [ 1/sqrt(2), 1/sqrt(3), 1/sqrt(2) ], + [ 0, 1/sqrt(3), 1/sqrt(2) ] +] + +A = A.transpose() + +[ + [ 1/sqrt(2), 1/sqrt(2), 0 ], + [ 1/sqrt(3), 1/sqrt(3), 1/sqrt(3) ], + [ 0, 1/sqrt(2), 1/sqrt(2) ] +] + +A = A.dot(deg_mat_inv_sqrt) + +[ + [ 1/sqrt(2) * 1/sqrt(2), 1/sqrt(2) * 1/sqrt(3), 0 ], + [ 1/sqrt(3) * 1/sqrt(2), 1/sqrt(3) * 1/sqrt(3), 1/sqrt(3) * 1/sqrt(2) ], + [ 0, 1/sqrt(2) * 1/sqrt(3), 1/sqrt(2) * 1/sqrt(2) ], +] + +thus: + +[ + [0.5 , 0.40824829, 0. ], + [0.40824829, 0.33333333, 0.40824829], + [0. , 0.40824829, 0.5 ] +] + +This checks out with the 1/sqrt(|N_{r}^{i}| |N_{r}^{j}|) formula. + +Then, we get back to the main calculation: + +y = x * W +y = A * y + +y = x * W + +[ + [ 1.25, 0.75 ], + [ 1.5 , 1.5 ], + [ 0.25, 0.75 ] +] + +y = A * y + +[ + 0.5 * [ 1.25, 0.75 ] + 0.40824829 * [ 1.5, 1.5 ], + 0.40824829 * [ 1.25, 0.75 ] + 0.33333333 * [ 1.5, 1.5 ] + 0.40824829 * [ 0.25, 0.75 ], + 0.40824829 * [ 1.5, 1.5 ] + 0.5 * [ 0.25, 0.75 ] +] + +that is: + +[ + [1.23737243, 0.98737244], + [1.11237243, 1.11237243], + [0.73737244, 0.98737244] +]. + +All checks out nicely, good. +""" + +from .dense import * +from .sparse import * +from .universal import * diff --git a/src/decagon_pytorch/convolve/dense.py b/src/decagon_pytorch/convolve/dense.py new file mode 100644 index 0000000..11e2b0d --- /dev/null +++ b/src/decagon_pytorch/convolve/dense.py @@ -0,0 +1,73 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import torch +from .dropout import dropout +from .weights import init_glorot +from typing import List, Callable + + +class DenseGraphConv(torch.nn.Module): + def __init__(self, in_channels: int, out_channels: int, + adjacency_matrix: torch.Tensor, **kwargs) -> None: + super().__init__(**kwargs) + self.in_channels = in_channels + self.out_channels = out_channels + self.weight = init_glorot(in_channels, out_channels) + self.adjacency_matrix = adjacency_matrix + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = torch.mm(x, self.weight) + x = torch.mm(self.adjacency_matrix, x) + return x + + +class DenseDropoutGraphConvActivation(torch.nn.Module): + def __init__(self, input_dim: int, output_dim: int, + adjacency_matrix: torch.Tensor, keep_prob: float=1., + activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, + **kwargs) -> None: + super().__init__(**kwargs) + self.graph_conv = DenseGraphConv(input_dim, output_dim, adjacency_matrix) + self.keep_prob = keep_prob + self.activation = activation + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = dropout(x, keep_prob=self.keep_prob) + x = self.graph_conv(x) + x = self.activation(x) + return x + + +class DenseMultiDGCA(torch.nn.Module): + def __init__(self, input_dim: List[int], output_dim: int, + adjacency_matrices: List[torch.Tensor], keep_prob: float=1., + activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, + **kwargs) -> None: + super().__init__(**kwargs) + self.input_dim = input_dim + self.output_dim = output_dim + self.adjacency_matrices = adjacency_matrices + self.keep_prob = keep_prob + self.activation = activation + self.dgca = None + self.build() + + def build(self): + if len(self.input_dim) != len(self.adjacency_matrices): + raise ValueError('input_dim must have the same length as adjacency_matrices') + self.dgca = [] + for input_dim, adj_mat in zip(self.input_dim, self.adjacency_matrices): + self.dgca.append(DenseDropoutGraphConvActivation(input_dim, self.output_dim, adj_mat, self.keep_prob, self.activation)) + + def forward(self, x: List[torch.Tensor]) -> List[torch.Tensor]: + if not isinstance(x, list): + raise ValueError('x must be a list of tensors') + out = torch.zeros(len(x[0]), self.output_dim, dtype=x[0].dtype) + for i, f in enumerate(self.dgca): + out += f(x[i]) + out = torch.nn.functional.normalize(out, p=2, dim=1) + return out diff --git a/src/decagon_pytorch/convolve/sparse.py b/src/decagon_pytorch/convolve/sparse.py new file mode 100644 index 0000000..7cbabf2 --- /dev/null +++ b/src/decagon_pytorch/convolve/sparse.py @@ -0,0 +1,78 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import torch +from .dropout import dropout_sparse +from .weights import init_glorot +from typing import List, Callable + + +class SparseGraphConv(torch.nn.Module): + """Convolution layer for sparse inputs.""" + def __init__(self, in_channels: int, out_channels: int, + adjacency_matrix: torch.Tensor, **kwargs) -> None: + super().__init__(**kwargs) + self.in_channels = in_channels + self.out_channels = out_channels + self.weight = init_glorot(in_channels, out_channels) + self.adjacency_matrix = adjacency_matrix + + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = torch.sparse.mm(x, self.weight) + x = torch.sparse.mm(self.adjacency_matrix, x) + return x + + +class SparseDropoutGraphConvActivation(torch.nn.Module): + def __init__(self, input_dim: int, output_dim: int, + adjacency_matrix: torch.Tensor, keep_prob: float=1., + activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, + **kwargs) -> None: + super().__init__(**kwargs) + self.input_dim = input_dim + self.output_dim = output_dim + self.adjacency_matrix = adjacency_matrix + self.keep_prob = keep_prob + self.activation = activation + self.sparse_graph_conv = SparseGraphConv(input_dim, output_dim, adjacency_matrix) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = dropout_sparse(x, self.keep_prob) + x = self.sparse_graph_conv(x) + x = self.activation(x) + return x + + +class SparseMultiDGCA(torch.nn.Module): + def __init__(self, input_dim: List[int], output_dim: int, + adjacency_matrices: List[torch.Tensor], keep_prob: float=1., + activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, + **kwargs) -> None: + super().__init__(**kwargs) + self.input_dim = input_dim + self.output_dim = output_dim + self.adjacency_matrices = adjacency_matrices + self.keep_prob = keep_prob + self.activation = activation + self.sparse_dgca = None + self.build() + + def build(self): + if len(self.input_dim) != len(self.adjacency_matrices): + raise ValueError('input_dim must have the same length as adjacency_matrices') + self.sparse_dgca = [] + for input_dim, adj_mat in zip(self.input_dim, self.adjacency_matrices): + self.sparse_dgca.append(SparseDropoutGraphConvActivation(input_dim, self.output_dim, adj_mat, self.keep_prob, self.activation)) + + def forward(self, x: List[torch.Tensor]) -> torch.Tensor: + if not isinstance(x, list): + raise ValueError('x must be a list of tensors') + out = torch.zeros(len(x[0]), self.output_dim, dtype=x[0].dtype) + for i, f in enumerate(self.sparse_dgca): + out += f(x[i]) + out = torch.nn.functional.normalize(out, p=2, dim=1) + return out diff --git a/src/decagon_pytorch/convolve/universal.py b/src/decagon_pytorch/convolve/universal.py new file mode 100644 index 0000000..0bcc8c3 --- /dev/null +++ b/src/decagon_pytorch/convolve/universal.py @@ -0,0 +1,85 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import torch +from .dropout import dropout_sparse, \ + dropout +from .weights import init_glorot +from typing import List, Callable + + +class GraphConv(torch.nn.Module): + """Convolution layer for sparse AND dense inputs.""" + def __init__(self, in_channels: int, out_channels: int, + adjacency_matrix: torch.Tensor, **kwargs) -> None: + super().__init__(**kwargs) + self.in_channels = in_channels + self.out_channels = out_channels + self.weight = init_glorot(in_channels, out_channels) + self.adjacency_matrix = adjacency_matrix + + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = torch.sparse.mm(x, self.weight) \ + if x.is_sparse \ + else torch.mm(x, self.weight) + x = torch.sparse.mm(self.adjacency_matrix, x) \ + if self.adjacency_matrix.is_sparse \ + else torch.mm(self.adjacency_matrix, x) + return x + + +class DropoutGraphConvActivation(torch.nn.Module): + def __init__(self, input_dim: int, output_dim: int, + adjacency_matrix: torch.Tensor, keep_prob: float=1., + activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, + **kwargs) -> None: + super().__init__(**kwargs) + self.input_dim = input_dim + self.output_dim = output_dim + self.adjacency_matrix = adjacency_matrix + self.keep_prob = keep_prob + self.activation = activation + self.graph_conv = GraphConv(input_dim, output_dim, adjacency_matrix) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = dropout_sparse(x, self.keep_prob) \ + if x.is_sparse \ + else dropout(x, self.keep_prob) + x = self.graph_conv(x) + x = self.activation(x) + return x + + +class MultiDGCA(torch.nn.Module): + def __init__(self, input_dim: List[int], output_dim: int, + adjacency_matrices: List[torch.Tensor], keep_prob: float=1., + activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, + **kwargs) -> None: + super().__init__(**kwargs) + self.input_dim = input_dim + self.output_dim = output_dim + self.adjacency_matrices = adjacency_matrices + self.keep_prob = keep_prob + self.activation = activation + self.dgca = None + self.build() + + def build(self): + if len(self.input_dim) != len(self.adjacency_matrices): + raise ValueError('input_dim must have the same length as adjacency_matrices') + self.dgca = [] + for input_dim, adj_mat in zip(self.input_dim, self.adjacency_matrices): + self.dgca.append(DenseDropoutGraphConvActivation(input_dim, self.output_dim, adj_mat, self.keep_prob, self.activation)) + + def forward(self, x: List[torch.Tensor]) -> List[torch.Tensor]: + if not isinstance(x, list): + raise ValueError('x must be a list of tensors') + out = torch.zeros(len(x[0]), self.output_dim, dtype=x[0].dtype) + for i, f in enumerate(self.dgca): + out += f(x[i]) + out = torch.nn.functional.normalize(out, p=2, dim=1) + return out -- 2.26.2 From 241a2f92b99e668efb3bf75de00a54392424be5e Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 29 May 2020 10:56:14 +0200 Subject: [PATCH 037/227] Fix tests. --- src/decagon_pytorch/convolve/dense.py | 4 +- src/decagon_pytorch/convolve/sparse.py | 4 +- src/decagon_pytorch/convolve/universal.py | 4 +- tests/decagon_pytorch/test_convolve.py | 55 ++++++++++++++--------- 4 files changed, 41 insertions(+), 26 deletions(-) diff --git a/src/decagon_pytorch/convolve/dense.py b/src/decagon_pytorch/convolve/dense.py index 11e2b0d..37cb700 100644 --- a/src/decagon_pytorch/convolve/dense.py +++ b/src/decagon_pytorch/convolve/dense.py @@ -5,8 +5,8 @@ import torch -from .dropout import dropout -from .weights import init_glorot +from ..dropout import dropout +from ..weights import init_glorot from typing import List, Callable diff --git a/src/decagon_pytorch/convolve/sparse.py b/src/decagon_pytorch/convolve/sparse.py index 7cbabf2..c472007 100644 --- a/src/decagon_pytorch/convolve/sparse.py +++ b/src/decagon_pytorch/convolve/sparse.py @@ -5,8 +5,8 @@ import torch -from .dropout import dropout_sparse -from .weights import init_glorot +from ..dropout import dropout_sparse +from ..weights import init_glorot from typing import List, Callable diff --git a/src/decagon_pytorch/convolve/universal.py b/src/decagon_pytorch/convolve/universal.py index 0bcc8c3..f39d1a8 100644 --- a/src/decagon_pytorch/convolve/universal.py +++ b/src/decagon_pytorch/convolve/universal.py @@ -5,9 +5,9 @@ import torch -from .dropout import dropout_sparse, \ +from ..dropout import dropout_sparse, \ dropout -from .weights import init_glorot +from ..weights import init_glorot from typing import List, Callable diff --git a/tests/decagon_pytorch/test_convolve.py b/tests/decagon_pytorch/test_convolve.py index 1f8b7a0..8dee490 100644 --- a/tests/decagon_pytorch/test_convolve.py +++ b/tests/decagon_pytorch/test_convolve.py @@ -179,24 +179,24 @@ def test_graph_conv(): assert np.all(latent_dense.detach().numpy() == latent_sparse.detach().numpy()) -def setup_function(fun): - if fun == test_dropout_graph_conv_activation or \ - fun == test_multi_dgca: - print('Disabling dropout for testing...') - setup_function.old_dropout = decagon_pytorch.convolve.dropout, \ - decagon_pytorch.convolve.dropout_sparse - - decagon_pytorch.convolve.dropout = lambda x, keep_prob: x - decagon_pytorch.convolve.dropout_sparse = lambda x, keep_prob: x - - -def teardown_function(fun): - print('Re-enabling dropout...') - if fun == test_dropout_graph_conv_activation or \ - fun == test_multi_dgca: - decagon_pytorch.convolve.dropout, \ - decagon_pytorch.convolve.dropout_sparse = \ - setup_function.old_dropout +# def setup_function(fun): +# if fun == test_dropout_graph_conv_activation or \ +# fun == test_multi_dgca: +# print('Disabling dropout for testing...') +# setup_function.old_dropout = decagon_pytorch.convolve.dropout, \ +# decagon_pytorch.convolve.dropout_sparse +# +# decagon_pytorch.convolve.dropout = lambda x, keep_prob: x +# decagon_pytorch.convolve.dropout_sparse = lambda x, keep_prob: x +# +# +# def teardown_function(fun): +# print('Re-enabling dropout...') +# if fun == test_dropout_graph_conv_activation or \ +# fun == test_multi_dgca: +# decagon_pytorch.convolve.dropout, \ +# decagon_pytorch.convolve.dropout_sparse = \ +# setup_function.old_dropout def flexible_dropout_graph_conv_activation_torch(keep_prob=1.): @@ -211,7 +211,20 @@ def flexible_dropout_graph_conv_activation_torch(keep_prob=1.): return latent -def test_dropout_graph_conv_activation(): +def _disable_dropout(monkeypatch): + monkeypatch.setattr(decagon_pytorch.convolve.dense, 'dropout', + lambda x, keep_prob: x) + monkeypatch.setattr(decagon_pytorch.convolve.sparse, 'dropout_sparse', + lambda x, keep_prob: x) + monkeypatch.setattr(decagon_pytorch.convolve.universal, 'dropout', + lambda x, keep_prob: x) + monkeypatch.setattr(decagon_pytorch.convolve.universal, 'dropout_sparse', + lambda x, keep_prob: x) + + +def test_dropout_graph_conv_activation(monkeypatch): + _disable_dropout(monkeypatch) + for i in range(11): keep_prob = i/10. if keep_prob == 0: @@ -243,7 +256,9 @@ def test_dropout_graph_conv_activation(): assert np.all(latent_sparse[nonzero] == latent_flex[nonzero]) -def test_multi_dgca(): +def test_multi_dgca(monkeypatch): + _disable_dropout(monkeypatch) + keep_prob = .5 torch.random.manual_seed(0) -- 2.26.2 From 52be4061b3adb364ba2fb5128178db45d9d7e7db Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 29 May 2020 11:33:17 +0200 Subject: [PATCH 038/227] Adjust DecagonLayer to normalize per edge type. --- src/decagon_pytorch/layer.py | 31 +++++++++++++++++++---------- tests/decagon_pytorch/test_layer.py | 29 ++++++++++++++++----------- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/decagon_pytorch/layer.py b/src/decagon_pytorch/layer.py index 69ca74c..d198328 100644 --- a/src/decagon_pytorch/layer.py +++ b/src/decagon_pytorch/layer.py @@ -124,11 +124,14 @@ class DecagonLayer(Layer): self.next_layer_repr = defaultdict(list) for (nt_row, nt_col), relation_types in self.data.relation_types.items(): + row_convs = [] + col_convs = [] + for rel in relation_types: conv = DropoutGraphConvActivation(self.input_dim[nt_col], self.output_dim[nt_row], rel.adjacency_matrix, self.keep_prob, self.rel_activation) - self.next_layer_repr[nt_row].append((conv, nt_col)) + row_convs.append(conv) if nt_row == nt_col: continue @@ -136,21 +139,27 @@ class DecagonLayer(Layer): conv = DropoutGraphConvActivation(self.input_dim[nt_row], self.output_dim[nt_col], rel.adjacency_matrix.transpose(0, 1), self.keep_prob, self.rel_activation) - self.next_layer_repr[nt_col].append((conv, nt_row)) + col_convs.append(conv) + + self.next_layer_repr[nt_row].append((row_convs, nt_col)) + + if nt_row == nt_col: + continue + + self.next_layer_repr[nt_col].append((col_convs, nt_row)) def __call__(self): prev_layer_repr = self.previous_layer() - next_layer_repr = [None] * len(self.data.node_types) + next_layer_repr = [ [] for _ in range(len(self.data.node_types)) ] print('next_layer_repr:', next_layer_repr) for i in range(len(self.data.node_types)): - next_layer_repr[i] = [ - conv(prev_layer_repr[neighbor_type]) \ - for (conv, neighbor_type) in \ - self.next_layer_repr[i] - ] + for convs, neighbor_type in self.next_layer_repr[i]: + convs = [ conv(prev_layer_repr[neighbor_type]) \ + for conv in convs ] + convs = sum(convs) + convs = torch.nn.functional.normalize(convs, p=2, dim=1) + next_layer_repr[i].append(convs) next_layer_repr[i] = sum(next_layer_repr[i]) - next_layer_repr[i] = torch.nn.functional.normalize(next_layer_repr[i], p=2, dim=1) - + next_layer_repr[i] = self.layer_activation(next_layer_repr[i]) print('next_layer_repr:', next_layer_repr) - # next_layer_repr = list(map(sum, next_layer_repr)) return next_layer_repr diff --git a/tests/decagon_pytorch/test_layer.py b/tests/decagon_pytorch/test_layer.py index cc0e5b7..5e7d6b0 100644 --- a/tests/decagon_pytorch/test_layer.py +++ b/tests/decagon_pytorch/test_layer.py @@ -136,16 +136,21 @@ def test_decagon_layer_03(): x = torch.tensor([-1, 0, 0.5, 1]) assert (d_layer.layer_activation(x) == torch.nn.functional.relu(x)).all() assert len(d_layer.next_layer_repr) == 2 - assert len(d_layer.next_layer_repr[0]) == 2 - assert len(d_layer.next_layer_repr[1]) == 4 - assert all(map(lambda a: isinstance(a[0], DropoutGraphConvActivation), - d_layer.next_layer_repr[0])) - assert all(map(lambda a: isinstance(a[0], DropoutGraphConvActivation), - d_layer.next_layer_repr[1])) - assert all(map(lambda a: a[0].output_dim == 32, - d_layer.next_layer_repr[0])) - assert all(map(lambda a: a[0].output_dim == 32, - d_layer.next_layer_repr[1])) + + for i in range(2): + assert len(d_layer.next_layer_repr[i]) == 2 + assert isinstance(d_layer.next_layer_repr[i], list) + assert isinstance(d_layer.next_layer_repr[i][0], tuple) + assert isinstance(d_layer.next_layer_repr[i][0][0], list) + assert isinstance(d_layer.next_layer_repr[i][0][1], int) + assert all([ + isinstance(dgca, DropoutGraphConvActivation) \ + for dgca in d_layer.next_layer_repr[i][0][0] + ]) + assert all([ + dgca.output_dim == 32 \ + for dgca in d_layer.next_layer_repr[i][0][0] + ]) def test_decagon_layer_04(): @@ -166,10 +171,10 @@ def test_decagon_layer_04(): keep_prob=1., rel_activation=lambda x: x, layer_activation=lambda x: x) - assert isinstance(d_layer.next_layer_repr[0][0][0], + assert isinstance(d_layer.next_layer_repr[0][0][0][0], DropoutGraphConvActivation) - weight = d_layer.next_layer_repr[0][0][0].graph_conv.weight + weight = d_layer.next_layer_repr[0][0][0][0].graph_conv.weight assert isinstance(weight, torch.Tensor) assert len(multi_dgca.sparse_dgca) == 1 -- 2.26.2 From 4ac42e5b3cef45f62e7ca0551ef743475a01248a Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 29 May 2020 11:47:33 +0200 Subject: [PATCH 039/227] Add test_decagon_layer_05(). --- tests/decagon_pytorch/test_layer.py | 50 +++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/decagon_pytorch/test_layer.py b/tests/decagon_pytorch/test_layer.py index 5e7d6b0..0b6207f 100644 --- a/tests/decagon_pytorch/test_layer.py +++ b/tests/decagon_pytorch/test_layer.py @@ -189,3 +189,53 @@ def test_decagon_layer_04(): assert len(out_d_layer) == 1 assert torch.all(out_d_layer[0] == out_multi_dgca) + + +def test_decagon_layer_05(): + # check if it is equivalent to MultiDGCA, as it should be + # this time for two relations, same edge type + + d = Data() + d.add_node_type('Dummy', 100) + d.add_relation_type('Dummy Relation 1', 0, 0, + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + d.add_relation_type('Dummy Relation 2', 0, 0, + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + + in_layer = OneHotInputLayer(d) + + multi_dgca = SparseMultiDGCA([100, 100], 32, + [r.adjacency_matrix for r in d.relation_types[0, 0]], + keep_prob=1., activation=lambda x: x) + + d_layer = DecagonLayer(d, in_layer, output_dim=32, + keep_prob=1., rel_activation=lambda x: x, + layer_activation=lambda x: x) + + assert all([ + isinstance(dgca, DropoutGraphConvActivation) \ + for dgca in d_layer.next_layer_repr[0][0][0] + ]) + + weight = [ dgca.graph_conv.weight \ + for dgca in d_layer.next_layer_repr[0][0][0] ] + assert all([ + isinstance(w, torch.Tensor) \ + for w in weight + ]) + + assert len(multi_dgca.sparse_dgca) == 2 + for i in range(2): + assert isinstance(multi_dgca.sparse_dgca[i], SparseDropoutGraphConvActivation) + + for i in range(2): + multi_dgca.sparse_dgca[i].sparse_graph_conv.weight = weight[i] + + out_d_layer = d_layer() + x = in_layer() + out_multi_dgca = multi_dgca([ x[0], x[0] ]) + + assert isinstance(out_d_layer, list) + assert len(out_d_layer) == 1 + + assert torch.all(out_d_layer[0] == out_multi_dgca) -- 2.26.2 From 1d2fac0919794d298e4a286e0c212060ca055e8b Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 29 May 2020 13:14:00 +0200 Subject: [PATCH 040/227] Split layer into submodules. --- src/decagon_pytorch/layer.py | 165 -------------------------- src/decagon_pytorch/layer/__init__.py | 26 ++++ src/decagon_pytorch/layer/convolve.py | 74 ++++++++++++ src/decagon_pytorch/layer/decode.py | 0 src/decagon_pytorch/layer/input.py | 65 ++++++++++ src/decagon_pytorch/layer/layer.py | 13 ++ 6 files changed, 178 insertions(+), 165 deletions(-) delete mode 100644 src/decagon_pytorch/layer.py create mode 100644 src/decagon_pytorch/layer/__init__.py create mode 100644 src/decagon_pytorch/layer/convolve.py create mode 100644 src/decagon_pytorch/layer/decode.py create mode 100644 src/decagon_pytorch/layer/input.py create mode 100644 src/decagon_pytorch/layer/layer.py diff --git a/src/decagon_pytorch/layer.py b/src/decagon_pytorch/layer.py deleted file mode 100644 index d198328..0000000 --- a/src/decagon_pytorch/layer.py +++ /dev/null @@ -1,165 +0,0 @@ -# -# This module implements a single layer of the Decagon -# model. This is going to be already quite complex, as -# we will be using all the graph convolutional building -# blocks. -# -# h_{i}^(k+1) = ϕ(∑_r ∑_{j∈N{r}^{i}} c_{r}^{ij} * \ -# W_{r}^(k) h_{j}^{k} + c_{r}^{i} h_{i}^(k)) -# -# N{r}^{i} - set of neighbors of node i under relation r -# W_{r}^(k) - relation-type specific weight matrix -# h_{i}^(k) - hidden state of node i in layer k -# h_{i}^(k)∈R^{d(k)} where d(k) is the dimensionality -# of the representation in k-th layer -# ϕ - activation function -# c_{r}^{ij} - normalization constants -# c_{r}^{ij} = 1/sqrt(|N_{r}^{i}| |N_{r}^{j}|) -# c_{r}^{i} - normalization constants -# c_{r}^{i} = 1/|N_{r}^{i}| -# - - -import torch -from .convolve import DropoutGraphConvActivation -from .data import Data -from typing import List, \ - Union, \ - Callable -from collections import defaultdict - - -class Layer(torch.nn.Module): - def __init__(self, - output_dim: Union[int, List[int]], - is_sparse: bool, - **kwargs) -> None: - super().__init__(**kwargs) - self.output_dim = output_dim - self.is_sparse = is_sparse - - -class InputLayer(Layer): - def __init__(self, data: Data, output_dim: Union[int, List[int]]= None, **kwargs) -> None: - output_dim = output_dim or \ - list(map(lambda a: a.count, data.node_types)) - if not isinstance(output_dim, list): - output_dim = [output_dim,] * len(data.node_types) - - super().__init__(output_dim, is_sparse=False, **kwargs) - self.data = data - self.node_reps = None - self.build() - - def build(self) -> None: - self.node_reps = [] - for i, nt in enumerate(self.data.node_types): - reps = torch.rand(nt.count, self.output_dim[i]) - reps = torch.nn.Parameter(reps) - self.register_parameter('node_reps[%d]' % i, reps) - self.node_reps.append(reps) - - def forward(self) -> List[torch.nn.Parameter]: - return self.node_reps - - def __repr__(self) -> str: - s = '' - s += 'GNN input layer with output_dim: %s\n' % self.output_dim - s += ' # of node types: %d\n' % len(self.data.node_types) - for nt in self.data.node_types: - s += ' - %s (%d)\n' % (nt.name, nt.count) - return s.strip() - - -class OneHotInputLayer(Layer): - def __init__(self, data: Data, **kwargs) -> None: - output_dim = [ a.count for a in data.node_types ] - super().__init__(output_dim, is_sparse=True, **kwargs) - self.data = data - self.node_reps = None - self.build() - - def build(self) -> None: - self.node_reps = [] - for i, nt in enumerate(self.data.node_types): - reps = torch.eye(nt.count).to_sparse() - reps = torch.nn.Parameter(reps) - self.register_parameter('node_reps[%d]' % i, reps) - self.node_reps.append(reps) - - def forward(self) -> List[torch.nn.Parameter]: - return self.node_reps - - def __repr__(self) -> str: - s = '' - s += 'One-hot GNN input layer\n' - s += ' # of node types: %d\n' % len(self.data.node_types) - for nt in self.data.node_types: - s += ' - %s (%d)\n' % (nt.name, nt.count) - return s.strip() - - -class DecagonLayer(Layer): - def __init__(self, - data: Data, - previous_layer: Layer, - output_dim: Union[int, List[int]], - keep_prob: float = 1., - rel_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, - layer_activation: Callable[[torch.Tensor], torch.Tensor] = torch.nn.functional.relu, - **kwargs): - if not isinstance(output_dim, list): - output_dim = [ output_dim ] * len(data.node_types) - super().__init__(output_dim, is_sparse=False, **kwargs) - self.data = data - self.previous_layer = previous_layer - self.input_dim = previous_layer.output_dim - self.keep_prob = keep_prob - self.rel_activation = rel_activation - self.layer_activation = layer_activation - self.next_layer_repr = None - self.build() - - def build(self): - self.next_layer_repr = defaultdict(list) - - for (nt_row, nt_col), relation_types in self.data.relation_types.items(): - row_convs = [] - col_convs = [] - - for rel in relation_types: - conv = DropoutGraphConvActivation(self.input_dim[nt_col], - self.output_dim[nt_row], rel.adjacency_matrix, - self.keep_prob, self.rel_activation) - row_convs.append(conv) - - if nt_row == nt_col: - continue - - conv = DropoutGraphConvActivation(self.input_dim[nt_row], - self.output_dim[nt_col], rel.adjacency_matrix.transpose(0, 1), - self.keep_prob, self.rel_activation) - col_convs.append(conv) - - self.next_layer_repr[nt_row].append((row_convs, nt_col)) - - if nt_row == nt_col: - continue - - self.next_layer_repr[nt_col].append((col_convs, nt_row)) - - def __call__(self): - prev_layer_repr = self.previous_layer() - next_layer_repr = [ [] for _ in range(len(self.data.node_types)) ] - print('next_layer_repr:', next_layer_repr) - for i in range(len(self.data.node_types)): - for convs, neighbor_type in self.next_layer_repr[i]: - convs = [ conv(prev_layer_repr[neighbor_type]) \ - for conv in convs ] - convs = sum(convs) - convs = torch.nn.functional.normalize(convs, p=2, dim=1) - next_layer_repr[i].append(convs) - next_layer_repr[i] = sum(next_layer_repr[i]) - next_layer_repr[i] = self.layer_activation(next_layer_repr[i]) - print('next_layer_repr:', next_layer_repr) - return next_layer_repr diff --git a/src/decagon_pytorch/layer/__init__.py b/src/decagon_pytorch/layer/__init__.py new file mode 100644 index 0000000..c1ae674 --- /dev/null +++ b/src/decagon_pytorch/layer/__init__.py @@ -0,0 +1,26 @@ +# +# This module implements a single layer of the Decagon +# model. This is going to be already quite complex, as +# we will be using all the graph convolutional building +# blocks. +# +# h_{i}^(k+1) = ϕ(∑_r ∑_{j∈N{r}^{i}} c_{r}^{ij} * \ +# W_{r}^(k) h_{j}^{k} + c_{r}^{i} h_{i}^(k)) +# +# N{r}^{i} - set of neighbors of node i under relation r +# W_{r}^(k) - relation-type specific weight matrix +# h_{i}^(k) - hidden state of node i in layer k +# h_{i}^(k)∈R^{d(k)} where d(k) is the dimensionality +# of the representation in k-th layer +# ϕ - activation function +# c_{r}^{ij} - normalization constants +# c_{r}^{ij} = 1/sqrt(|N_{r}^{i}| |N_{r}^{j}|) +# c_{r}^{i} - normalization constants +# c_{r}^{i} = 1/|N_{r}^{i}| +# + + +from .layer import * +from .input import * +from .convolve import * +from .decode import * diff --git a/src/decagon_pytorch/layer/convolve.py b/src/decagon_pytorch/layer/convolve.py new file mode 100644 index 0000000..8229f36 --- /dev/null +++ b/src/decagon_pytorch/layer/convolve.py @@ -0,0 +1,74 @@ +from .layer import Layer +import torch +from ..convolve import DropoutGraphConvActivation +from ..data import Data +from typing import List, \ + Union, \ + Callable +from collections import defaultdict + + +class DecagonLayer(Layer): + def __init__(self, + data: Data, + previous_layer: Layer, + output_dim: Union[int, List[int]], + keep_prob: float = 1., + rel_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, + layer_activation: Callable[[torch.Tensor], torch.Tensor] = torch.nn.functional.relu, + **kwargs): + if not isinstance(output_dim, list): + output_dim = [ output_dim ] * len(data.node_types) + super().__init__(output_dim, is_sparse=False, **kwargs) + self.data = data + self.previous_layer = previous_layer + self.input_dim = previous_layer.output_dim + self.keep_prob = keep_prob + self.rel_activation = rel_activation + self.layer_activation = layer_activation + self.next_layer_repr = None + self.build() + + def build(self): + self.next_layer_repr = defaultdict(list) + + for (nt_row, nt_col), relation_types in self.data.relation_types.items(): + row_convs = [] + col_convs = [] + + for rel in relation_types: + conv = DropoutGraphConvActivation(self.input_dim[nt_col], + self.output_dim[nt_row], rel.adjacency_matrix, + self.keep_prob, self.rel_activation) + row_convs.append(conv) + + if nt_row == nt_col: + continue + + conv = DropoutGraphConvActivation(self.input_dim[nt_row], + self.output_dim[nt_col], rel.adjacency_matrix.transpose(0, 1), + self.keep_prob, self.rel_activation) + col_convs.append(conv) + + self.next_layer_repr[nt_row].append((row_convs, nt_col)) + + if nt_row == nt_col: + continue + + self.next_layer_repr[nt_col].append((col_convs, nt_row)) + + def __call__(self): + prev_layer_repr = self.previous_layer() + next_layer_repr = [ [] for _ in range(len(self.data.node_types)) ] + print('next_layer_repr:', next_layer_repr) + for i in range(len(self.data.node_types)): + for convs, neighbor_type in self.next_layer_repr[i]: + convs = [ conv(prev_layer_repr[neighbor_type]) \ + for conv in convs ] + convs = sum(convs) + convs = torch.nn.functional.normalize(convs, p=2, dim=1) + next_layer_repr[i].append(convs) + next_layer_repr[i] = sum(next_layer_repr[i]) + next_layer_repr[i] = self.layer_activation(next_layer_repr[i]) + print('next_layer_repr:', next_layer_repr) + return next_layer_repr diff --git a/src/decagon_pytorch/layer/decode.py b/src/decagon_pytorch/layer/decode.py new file mode 100644 index 0000000..e69de29 diff --git a/src/decagon_pytorch/layer/input.py b/src/decagon_pytorch/layer/input.py new file mode 100644 index 0000000..f794277 --- /dev/null +++ b/src/decagon_pytorch/layer/input.py @@ -0,0 +1,65 @@ +from .layer import Layer +import torch +from typing import Union, \ + List +from ..data import Data + + +class InputLayer(Layer): + def __init__(self, data: Data, output_dim: Union[int, List[int]]= None, **kwargs) -> None: + output_dim = output_dim or \ + list(map(lambda a: a.count, data.node_types)) + if not isinstance(output_dim, list): + output_dim = [output_dim,] * len(data.node_types) + + super().__init__(output_dim, is_sparse=False, **kwargs) + self.data = data + self.node_reps = None + self.build() + + def build(self) -> None: + self.node_reps = [] + for i, nt in enumerate(self.data.node_types): + reps = torch.rand(nt.count, self.output_dim[i]) + reps = torch.nn.Parameter(reps) + self.register_parameter('node_reps[%d]' % i, reps) + self.node_reps.append(reps) + + def forward(self) -> List[torch.nn.Parameter]: + return self.node_reps + + def __repr__(self) -> str: + s = '' + s += 'GNN input layer with output_dim: %s\n' % self.output_dim + s += ' # of node types: %d\n' % len(self.data.node_types) + for nt in self.data.node_types: + s += ' - %s (%d)\n' % (nt.name, nt.count) + return s.strip() + + +class OneHotInputLayer(Layer): + def __init__(self, data: Data, **kwargs) -> None: + output_dim = [ a.count for a in data.node_types ] + super().__init__(output_dim, is_sparse=True, **kwargs) + self.data = data + self.node_reps = None + self.build() + + def build(self) -> None: + self.node_reps = [] + for i, nt in enumerate(self.data.node_types): + reps = torch.eye(nt.count).to_sparse() + reps = torch.nn.Parameter(reps) + self.register_parameter('node_reps[%d]' % i, reps) + self.node_reps.append(reps) + + def forward(self) -> List[torch.nn.Parameter]: + return self.node_reps + + def __repr__(self) -> str: + s = '' + s += 'One-hot GNN input layer\n' + s += ' # of node types: %d\n' % len(self.data.node_types) + for nt in self.data.node_types: + s += ' - %s (%d)\n' % (nt.name, nt.count) + return s.strip() diff --git a/src/decagon_pytorch/layer/layer.py b/src/decagon_pytorch/layer/layer.py new file mode 100644 index 0000000..91f705c --- /dev/null +++ b/src/decagon_pytorch/layer/layer.py @@ -0,0 +1,13 @@ +import torch +from typing import List, \ + Union + + +class Layer(torch.nn.Module): + def __init__(self, + output_dim: Union[int, List[int]], + is_sparse: bool, + **kwargs) -> None: + super().__init__(**kwargs) + self.output_dim = output_dim + self.is_sparse = is_sparse -- 2.26.2 From c37e4dc01e0623a3a6f6dd1490335a0edd3e4624 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 29 May 2020 15:23:58 +0200 Subject: [PATCH 041/227] Ok the very first decoding seems to work end-to-end. --- src/decagon_pytorch/__init__.py | 1 + src/decagon_pytorch/layer/decode.py | 60 +++++++ tests/decagon_pytorch/test_layer.py | 241 ---------------------------- 3 files changed, 61 insertions(+), 241 deletions(-) delete mode 100644 tests/decagon_pytorch/test_layer.py diff --git a/src/decagon_pytorch/__init__.py b/src/decagon_pytorch/__init__.py index f628a28..bc82355 100644 --- a/src/decagon_pytorch/__init__.py +++ b/src/decagon_pytorch/__init__.py @@ -1,3 +1,4 @@ from .weights import * from .convolve import * from .model import * +from .layer import * diff --git a/src/decagon_pytorch/layer/decode.py b/src/decagon_pytorch/layer/decode.py index e69de29..d0d548b 100644 --- a/src/decagon_pytorch/layer/decode.py +++ b/src/decagon_pytorch/layer/decode.py @@ -0,0 +1,60 @@ +from .layer import Layer +import torch +from ..data import Data +from typing import Type, \ + List, \ + Callable, \ + Union, \ + Dict, \ + Tuple +from ..decode import DEDICOMDecoder + + +class DecodeLayer(torch.nn.Module): + def __init__(self, + data: Data, + last_layer: Layer, + keep_prob: float = 1., + activation: Callable[[torch.Tensor], torch.Tensor] = torch.sigmoid, + decoder_class: Union[Type, Dict[Tuple[int, int], Type]] = DEDICOMDecoder, **kwargs) -> None: + + super().__init__(**kwargs) + self.data = data + self.last_layer = last_layer + self.keep_prob = keep_prob + self.activation = activation + assert all([a == last_layer.output_dim[0] \ + for a in last_layer.output_dim]) + self.input_dim = last_layer.output_dim[0] + self.output_dim = 1 + self.decoder_class = decoder_class + self.decoders = None + self.build() + + def build(self) -> None: + self.decoders = {} + for (node_type_row, node_type_col), rels in self.data.relation_types.items(): + key = (node_type_row, node_type_col) + if isinstance(self.decoder_class, dict): + if key in self.decoder_class: + decoder_class = self.decoder_class[key] + else: + raise KeyError('Decoder not specified for edge type: %d -- %d' % key) + else: + decoder_class = self.decoder_class + + self.decoders[key] = decoder_class(self.input_dim, + num_relation_types = len(rels), + drop_prob = 1. - self.keep_prob, + activation = self.activation) + + + def forward(self, last_layer_repr: List[torch.Tensor]): + res = {} + for (node_type_row, node_type_col), rel in self.data.relation_types.items(): + key = (node_type_row, node_type_col) + inputs_row = last_layer_repr[node_type_row] + inputs_col = last_layer_repr[node_type_col] + pred_adj_matrices = self.decoders[key](inputs_row, inputs_col) + res[node_type_row, node_type_col] = pred_adj_matrices + return res diff --git a/tests/decagon_pytorch/test_layer.py b/tests/decagon_pytorch/test_layer.py deleted file mode 100644 index 0b6207f..0000000 --- a/tests/decagon_pytorch/test_layer.py +++ /dev/null @@ -1,241 +0,0 @@ -from decagon_pytorch.layer import InputLayer, \ - OneHotInputLayer, \ - DecagonLayer -from decagon_pytorch.data import Data -import torch -import pytest -from decagon_pytorch.convolve import SparseDropoutGraphConvActivation, \ - SparseMultiDGCA, \ - DropoutGraphConvActivation - - -def _some_data(): - d = Data() - d.add_node_type('Gene', 1000) - d.add_node_type('Drug', 100) - d.add_relation_type('Target', 1, 0, None) - d.add_relation_type('Interaction', 0, 0, None) - d.add_relation_type('Side Effect: Nausea', 1, 1, None) - d.add_relation_type('Side Effect: Infertility', 1, 1, None) - d.add_relation_type('Side Effect: Death', 1, 1, None) - return d - - -def _some_data_with_interactions(): - d = Data() - d.add_node_type('Gene', 1000) - d.add_node_type('Drug', 100) - d.add_relation_type('Target', 1, 0, - torch.rand((100, 1000), dtype=torch.float32).round()) - d.add_relation_type('Interaction', 0, 0, - torch.rand((1000, 1000), dtype=torch.float32).round()) - d.add_relation_type('Side Effect: Nausea', 1, 1, - torch.rand((100, 100), dtype=torch.float32).round()) - d.add_relation_type('Side Effect: Infertility', 1, 1, - torch.rand((100, 100), dtype=torch.float32).round()) - d.add_relation_type('Side Effect: Death', 1, 1, - torch.rand((100, 100), dtype=torch.float32).round()) - return d - - -def test_input_layer_01(): - d = _some_data() - for output_dim in [32, 64, 128]: - layer = InputLayer(d, output_dim) - assert layer.output_dim[0] == output_dim - assert len(layer.node_reps) == 2 - assert layer.node_reps[0].shape == (1000, output_dim) - assert layer.node_reps[1].shape == (100, output_dim) - assert layer.data == d - - -def test_input_layer_02(): - d = _some_data() - layer = InputLayer(d, 32) - res = layer() - assert isinstance(res[0], torch.Tensor) - assert isinstance(res[1], torch.Tensor) - assert res[0].shape == (1000, 32) - assert res[1].shape == (100, 32) - assert torch.all(res[0] == layer.node_reps[0]) - assert torch.all(res[1] == layer.node_reps[1]) - - -def test_input_layer_03(): - if torch.cuda.device_count() == 0: - pytest.skip('No CUDA devices on this host') - d = _some_data() - layer = InputLayer(d, 32) - device = torch.device('cuda:0') - layer = layer.to(device) - print(list(layer.parameters())) - # assert layer.device.type == 'cuda:0' - assert layer.node_reps[0].device == device - assert layer.node_reps[1].device == device - - -def test_one_hot_input_layer_01(): - d = _some_data() - layer = OneHotInputLayer(d) - assert layer.output_dim == [1000, 100] - assert len(layer.node_reps) == 2 - assert layer.node_reps[0].shape == (1000, 1000) - assert layer.node_reps[1].shape == (100, 100) - assert layer.data == d - assert layer.is_sparse - - -def test_one_hot_input_layer_02(): - d = _some_data() - layer = OneHotInputLayer(d) - res = layer() - assert isinstance(res[0], torch.Tensor) - assert isinstance(res[1], torch.Tensor) - assert res[0].shape == (1000, 1000) - assert res[1].shape == (100, 100) - assert torch.all(res[0].to_dense() == layer.node_reps[0].to_dense()) - assert torch.all(res[1].to_dense() == layer.node_reps[1].to_dense()) - - -def test_one_hot_input_layer_03(): - if torch.cuda.device_count() == 0: - pytest.skip('No CUDA devices on this host') - d = _some_data() - layer = OneHotInputLayer(d) - device = torch.device('cuda:0') - layer = layer.to(device) - print(list(layer.parameters())) - # assert layer.device.type == 'cuda:0' - assert layer.node_reps[0].device == device - assert layer.node_reps[1].device == device - - -def test_decagon_layer_01(): - d = _some_data_with_interactions() - in_layer = InputLayer(d) - d_layer = DecagonLayer(d, in_layer, output_dim=32) - - -def test_decagon_layer_02(): - d = _some_data_with_interactions() - in_layer = OneHotInputLayer(d) - d_layer = DecagonLayer(d, in_layer, output_dim=32) - _ = d_layer() # dummy call - - -def test_decagon_layer_03(): - d = _some_data_with_interactions() - in_layer = OneHotInputLayer(d) - d_layer = DecagonLayer(d, in_layer, output_dim=32) - assert d_layer.data == d - assert d_layer.previous_layer == in_layer - assert d_layer.input_dim == [ 1000, 100 ] - assert not d_layer.is_sparse - assert d_layer.keep_prob == 1. - assert d_layer.rel_activation(0.5) == 0.5 - x = torch.tensor([-1, 0, 0.5, 1]) - assert (d_layer.layer_activation(x) == torch.nn.functional.relu(x)).all() - assert len(d_layer.next_layer_repr) == 2 - - for i in range(2): - assert len(d_layer.next_layer_repr[i]) == 2 - assert isinstance(d_layer.next_layer_repr[i], list) - assert isinstance(d_layer.next_layer_repr[i][0], tuple) - assert isinstance(d_layer.next_layer_repr[i][0][0], list) - assert isinstance(d_layer.next_layer_repr[i][0][1], int) - assert all([ - isinstance(dgca, DropoutGraphConvActivation) \ - for dgca in d_layer.next_layer_repr[i][0][0] - ]) - assert all([ - dgca.output_dim == 32 \ - for dgca in d_layer.next_layer_repr[i][0][0] - ]) - - -def test_decagon_layer_04(): - # check if it is equivalent to MultiDGCA, as it should be - - d = Data() - d.add_node_type('Dummy', 100) - d.add_relation_type('Dummy Relation', 0, 0, - torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) - - in_layer = OneHotInputLayer(d) - - multi_dgca = SparseMultiDGCA([10], 32, - [r.adjacency_matrix for r in d.relation_types[0, 0]], - keep_prob=1., activation=lambda x: x) - - d_layer = DecagonLayer(d, in_layer, output_dim=32, - keep_prob=1., rel_activation=lambda x: x, - layer_activation=lambda x: x) - - assert isinstance(d_layer.next_layer_repr[0][0][0][0], - DropoutGraphConvActivation) - - weight = d_layer.next_layer_repr[0][0][0][0].graph_conv.weight - assert isinstance(weight, torch.Tensor) - - assert len(multi_dgca.sparse_dgca) == 1 - assert isinstance(multi_dgca.sparse_dgca[0], SparseDropoutGraphConvActivation) - - multi_dgca.sparse_dgca[0].sparse_graph_conv.weight = weight - - out_d_layer = d_layer() - out_multi_dgca = multi_dgca(in_layer()) - - assert isinstance(out_d_layer, list) - assert len(out_d_layer) == 1 - - assert torch.all(out_d_layer[0] == out_multi_dgca) - - -def test_decagon_layer_05(): - # check if it is equivalent to MultiDGCA, as it should be - # this time for two relations, same edge type - - d = Data() - d.add_node_type('Dummy', 100) - d.add_relation_type('Dummy Relation 1', 0, 0, - torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) - d.add_relation_type('Dummy Relation 2', 0, 0, - torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) - - in_layer = OneHotInputLayer(d) - - multi_dgca = SparseMultiDGCA([100, 100], 32, - [r.adjacency_matrix for r in d.relation_types[0, 0]], - keep_prob=1., activation=lambda x: x) - - d_layer = DecagonLayer(d, in_layer, output_dim=32, - keep_prob=1., rel_activation=lambda x: x, - layer_activation=lambda x: x) - - assert all([ - isinstance(dgca, DropoutGraphConvActivation) \ - for dgca in d_layer.next_layer_repr[0][0][0] - ]) - - weight = [ dgca.graph_conv.weight \ - for dgca in d_layer.next_layer_repr[0][0][0] ] - assert all([ - isinstance(w, torch.Tensor) \ - for w in weight - ]) - - assert len(multi_dgca.sparse_dgca) == 2 - for i in range(2): - assert isinstance(multi_dgca.sparse_dgca[i], SparseDropoutGraphConvActivation) - - for i in range(2): - multi_dgca.sparse_dgca[i].sparse_graph_conv.weight = weight[i] - - out_d_layer = d_layer() - x = in_layer() - out_multi_dgca = multi_dgca([ x[0], x[0] ]) - - assert isinstance(out_d_layer, list) - assert len(out_d_layer) == 1 - - assert torch.all(out_d_layer[0] == out_multi_dgca) -- 2.26.2 From 0f1afa3d2cfa27b2282e946d920617d3899fc714 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 29 May 2020 15:24:46 +0200 Subject: [PATCH 042/227] Put back layer tests. --- .../layer/test_layer_convolve.py | 169 ++++++++++++++++++ .../layer/test_layer_decode.py | 22 +++ .../decagon_pytorch/layer/test_layer_input.py | 110 ++++++++++++ 3 files changed, 301 insertions(+) create mode 100644 tests/decagon_pytorch/layer/test_layer_convolve.py create mode 100644 tests/decagon_pytorch/layer/test_layer_decode.py create mode 100644 tests/decagon_pytorch/layer/test_layer_input.py diff --git a/tests/decagon_pytorch/layer/test_layer_convolve.py b/tests/decagon_pytorch/layer/test_layer_convolve.py new file mode 100644 index 0000000..69e3a75 --- /dev/null +++ b/tests/decagon_pytorch/layer/test_layer_convolve.py @@ -0,0 +1,169 @@ +from decagon_pytorch.layer import InputLayer, \ + OneHotInputLayer, \ + DecagonLayer +from decagon_pytorch.data import Data +import torch +import pytest +from decagon_pytorch.convolve import SparseDropoutGraphConvActivation, \ + SparseMultiDGCA, \ + DropoutGraphConvActivation + + +def _some_data(): + d = Data() + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + d.add_relation_type('Target', 1, 0, None) + d.add_relation_type('Interaction', 0, 0, None) + d.add_relation_type('Side Effect: Nausea', 1, 1, None) + d.add_relation_type('Side Effect: Infertility', 1, 1, None) + d.add_relation_type('Side Effect: Death', 1, 1, None) + return d + + +def _some_data_with_interactions(): + d = Data() + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + d.add_relation_type('Target', 1, 0, + torch.rand((100, 1000), dtype=torch.float32).round()) + d.add_relation_type('Interaction', 0, 0, + torch.rand((1000, 1000), dtype=torch.float32).round()) + d.add_relation_type('Side Effect: Nausea', 1, 1, + torch.rand((100, 100), dtype=torch.float32).round()) + d.add_relation_type('Side Effect: Infertility', 1, 1, + torch.rand((100, 100), dtype=torch.float32).round()) + d.add_relation_type('Side Effect: Death', 1, 1, + torch.rand((100, 100), dtype=torch.float32).round()) + return d + + +def test_decagon_layer_01(): + d = _some_data_with_interactions() + in_layer = InputLayer(d) + d_layer = DecagonLayer(d, in_layer, output_dim=32) + + +def test_decagon_layer_02(): + d = _some_data_with_interactions() + in_layer = OneHotInputLayer(d) + d_layer = DecagonLayer(d, in_layer, output_dim=32) + _ = d_layer() # dummy call + + +def test_decagon_layer_03(): + d = _some_data_with_interactions() + in_layer = OneHotInputLayer(d) + d_layer = DecagonLayer(d, in_layer, output_dim=32) + assert d_layer.data == d + assert d_layer.previous_layer == in_layer + assert d_layer.input_dim == [ 1000, 100 ] + assert not d_layer.is_sparse + assert d_layer.keep_prob == 1. + assert d_layer.rel_activation(0.5) == 0.5 + x = torch.tensor([-1, 0, 0.5, 1]) + assert (d_layer.layer_activation(x) == torch.nn.functional.relu(x)).all() + assert len(d_layer.next_layer_repr) == 2 + + for i in range(2): + assert len(d_layer.next_layer_repr[i]) == 2 + assert isinstance(d_layer.next_layer_repr[i], list) + assert isinstance(d_layer.next_layer_repr[i][0], tuple) + assert isinstance(d_layer.next_layer_repr[i][0][0], list) + assert isinstance(d_layer.next_layer_repr[i][0][1], int) + assert all([ + isinstance(dgca, DropoutGraphConvActivation) \ + for dgca in d_layer.next_layer_repr[i][0][0] + ]) + assert all([ + dgca.output_dim == 32 \ + for dgca in d_layer.next_layer_repr[i][0][0] + ]) + + +def test_decagon_layer_04(): + # check if it is equivalent to MultiDGCA, as it should be + + d = Data() + d.add_node_type('Dummy', 100) + d.add_relation_type('Dummy Relation', 0, 0, + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + + in_layer = OneHotInputLayer(d) + + multi_dgca = SparseMultiDGCA([10], 32, + [r.adjacency_matrix for r in d.relation_types[0, 0]], + keep_prob=1., activation=lambda x: x) + + d_layer = DecagonLayer(d, in_layer, output_dim=32, + keep_prob=1., rel_activation=lambda x: x, + layer_activation=lambda x: x) + + assert isinstance(d_layer.next_layer_repr[0][0][0][0], + DropoutGraphConvActivation) + + weight = d_layer.next_layer_repr[0][0][0][0].graph_conv.weight + assert isinstance(weight, torch.Tensor) + + assert len(multi_dgca.sparse_dgca) == 1 + assert isinstance(multi_dgca.sparse_dgca[0], SparseDropoutGraphConvActivation) + + multi_dgca.sparse_dgca[0].sparse_graph_conv.weight = weight + + out_d_layer = d_layer() + out_multi_dgca = multi_dgca(in_layer()) + + assert isinstance(out_d_layer, list) + assert len(out_d_layer) == 1 + + assert torch.all(out_d_layer[0] == out_multi_dgca) + + +def test_decagon_layer_05(): + # check if it is equivalent to MultiDGCA, as it should be + # this time for two relations, same edge type + + d = Data() + d.add_node_type('Dummy', 100) + d.add_relation_type('Dummy Relation 1', 0, 0, + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + d.add_relation_type('Dummy Relation 2', 0, 0, + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + + in_layer = OneHotInputLayer(d) + + multi_dgca = SparseMultiDGCA([100, 100], 32, + [r.adjacency_matrix for r in d.relation_types[0, 0]], + keep_prob=1., activation=lambda x: x) + + d_layer = DecagonLayer(d, in_layer, output_dim=32, + keep_prob=1., rel_activation=lambda x: x, + layer_activation=lambda x: x) + + assert all([ + isinstance(dgca, DropoutGraphConvActivation) \ + for dgca in d_layer.next_layer_repr[0][0][0] + ]) + + weight = [ dgca.graph_conv.weight \ + for dgca in d_layer.next_layer_repr[0][0][0] ] + assert all([ + isinstance(w, torch.Tensor) \ + for w in weight + ]) + + assert len(multi_dgca.sparse_dgca) == 2 + for i in range(2): + assert isinstance(multi_dgca.sparse_dgca[i], SparseDropoutGraphConvActivation) + + for i in range(2): + multi_dgca.sparse_dgca[i].sparse_graph_conv.weight = weight[i] + + out_d_layer = d_layer() + x = in_layer() + out_multi_dgca = multi_dgca([ x[0], x[0] ]) + + assert isinstance(out_d_layer, list) + assert len(out_d_layer) == 1 + + assert torch.all(out_d_layer[0] == out_multi_dgca) diff --git a/tests/decagon_pytorch/layer/test_layer_decode.py b/tests/decagon_pytorch/layer/test_layer_decode.py new file mode 100644 index 0000000..57c554b --- /dev/null +++ b/tests/decagon_pytorch/layer/test_layer_decode.py @@ -0,0 +1,22 @@ +from decagon_pytorch.layer import OneHotInputLayer, \ + DecagonLayer, \ + DecodeLayer +from decagon_pytorch.decode import DEDICOMDecoder +from decagon_pytorch.data import Data +import torch + + +def test_decode_layer_01(): + d = Data() + d.add_node_type('Dummy', 100) + d.add_relation_type('Dummy Relation 1', 0, 0, + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + in_layer = OneHotInputLayer(d) + d_layer = DecagonLayer(d, in_layer, 32) + last_layer_repr = d_layer() + dec = DecodeLayer(d, last_layer = d_layer, decoder_class = DEDICOMDecoder) + pred_adj_matrices = dec(last_layer_repr) + assert isinstance(pred_adj_matrices, dict) + assert len(pred_adj_matrices) == 1 + assert isinstance(pred_adj_matrices[0, 0], list) + assert len(pred_adj_matrices[0, 0]) == 1 diff --git a/tests/decagon_pytorch/layer/test_layer_input.py b/tests/decagon_pytorch/layer/test_layer_input.py new file mode 100644 index 0000000..27b84e7 --- /dev/null +++ b/tests/decagon_pytorch/layer/test_layer_input.py @@ -0,0 +1,110 @@ +from decagon_pytorch.layer import InputLayer, \ + OneHotInputLayer, \ + DecagonLayer +from decagon_pytorch.data import Data +import torch +import pytest +from decagon_pytorch.convolve import SparseDropoutGraphConvActivation, \ + SparseMultiDGCA, \ + DropoutGraphConvActivation + + +def _some_data(): + d = Data() + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + d.add_relation_type('Target', 1, 0, None) + d.add_relation_type('Interaction', 0, 0, None) + d.add_relation_type('Side Effect: Nausea', 1, 1, None) + d.add_relation_type('Side Effect: Infertility', 1, 1, None) + d.add_relation_type('Side Effect: Death', 1, 1, None) + return d + + +def _some_data_with_interactions(): + d = Data() + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + d.add_relation_type('Target', 1, 0, + torch.rand((100, 1000), dtype=torch.float32).round()) + d.add_relation_type('Interaction', 0, 0, + torch.rand((1000, 1000), dtype=torch.float32).round()) + d.add_relation_type('Side Effect: Nausea', 1, 1, + torch.rand((100, 100), dtype=torch.float32).round()) + d.add_relation_type('Side Effect: Infertility', 1, 1, + torch.rand((100, 100), dtype=torch.float32).round()) + d.add_relation_type('Side Effect: Death', 1, 1, + torch.rand((100, 100), dtype=torch.float32).round()) + return d + + +def test_input_layer_01(): + d = _some_data() + for output_dim in [32, 64, 128]: + layer = InputLayer(d, output_dim) + assert layer.output_dim[0] == output_dim + assert len(layer.node_reps) == 2 + assert layer.node_reps[0].shape == (1000, output_dim) + assert layer.node_reps[1].shape == (100, output_dim) + assert layer.data == d + + +def test_input_layer_02(): + d = _some_data() + layer = InputLayer(d, 32) + res = layer() + assert isinstance(res[0], torch.Tensor) + assert isinstance(res[1], torch.Tensor) + assert res[0].shape == (1000, 32) + assert res[1].shape == (100, 32) + assert torch.all(res[0] == layer.node_reps[0]) + assert torch.all(res[1] == layer.node_reps[1]) + + +def test_input_layer_03(): + if torch.cuda.device_count() == 0: + pytest.skip('No CUDA devices on this host') + d = _some_data() + layer = InputLayer(d, 32) + device = torch.device('cuda:0') + layer = layer.to(device) + print(list(layer.parameters())) + # assert layer.device.type == 'cuda:0' + assert layer.node_reps[0].device == device + assert layer.node_reps[1].device == device + + +def test_one_hot_input_layer_01(): + d = _some_data() + layer = OneHotInputLayer(d) + assert layer.output_dim == [1000, 100] + assert len(layer.node_reps) == 2 + assert layer.node_reps[0].shape == (1000, 1000) + assert layer.node_reps[1].shape == (100, 100) + assert layer.data == d + assert layer.is_sparse + + +def test_one_hot_input_layer_02(): + d = _some_data() + layer = OneHotInputLayer(d) + res = layer() + assert isinstance(res[0], torch.Tensor) + assert isinstance(res[1], torch.Tensor) + assert res[0].shape == (1000, 1000) + assert res[1].shape == (100, 100) + assert torch.all(res[0].to_dense() == layer.node_reps[0].to_dense()) + assert torch.all(res[1].to_dense() == layer.node_reps[1].to_dense()) + + +def test_one_hot_input_layer_03(): + if torch.cuda.device_count() == 0: + pytest.skip('No CUDA devices on this host') + d = _some_data() + layer = OneHotInputLayer(d) + device = torch.device('cuda:0') + layer = layer.to(device) + print(list(layer.parameters())) + # assert layer.device.type == 'cuda:0' + assert layer.node_reps[0].device == device + assert layer.node_reps[1].device == device -- 2.26.2 From 4f243fbc045f5c4ada6d8cd7ceb89f8d3c85e922 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 29 May 2020 15:28:30 +0200 Subject: [PATCH 043/227] Copyright notes. --- src/decagon_pytorch/__init__.py | 5 +++++ src/decagon_pytorch/batch.py | 6 ++++++ src/decagon_pytorch/convolve/__init__.py | 1 + src/decagon_pytorch/data.py | 6 ++++++ src/decagon_pytorch/decode.py | 6 ++++++ src/decagon_pytorch/dropout.py | 6 ++++++ src/decagon_pytorch/layer/__init__.py | 6 ++++++ src/decagon_pytorch/layer/convolve.py | 6 ++++++ src/decagon_pytorch/layer/decode.py | 6 ++++++ src/decagon_pytorch/layer/input.py | 6 ++++++ src/decagon_pytorch/layer/layer.py | 6 ++++++ src/decagon_pytorch/model.py | 6 ++++++ src/decagon_pytorch/normalize.py | 6 ++++++ src/decagon_pytorch/weights.py | 6 ++++++ 14 files changed, 78 insertions(+) diff --git a/src/decagon_pytorch/__init__.py b/src/decagon_pytorch/__init__.py index bc82355..f95e8e8 100644 --- a/src/decagon_pytorch/__init__.py +++ b/src/decagon_pytorch/__init__.py @@ -1,3 +1,8 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + from .weights import * from .convolve import * from .model import * diff --git a/src/decagon_pytorch/batch.py b/src/decagon_pytorch/batch.py index 8be3fe6..aeed492 100644 --- a/src/decagon_pytorch/batch.py +++ b/src/decagon_pytorch/batch.py @@ -1,3 +1,9 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + import scipy.sparse as sp diff --git a/src/decagon_pytorch/convolve/__init__.py b/src/decagon_pytorch/convolve/__init__.py index f3c2e40..c0f6305 100644 --- a/src/decagon_pytorch/convolve/__init__.py +++ b/src/decagon_pytorch/convolve/__init__.py @@ -3,6 +3,7 @@ # License: GPLv3 # + """ This module implements the basic convolutional blocks of Decagon. Just as a quick reminder, the basic convolution formula here is: diff --git a/src/decagon_pytorch/data.py b/src/decagon_pytorch/data.py index 2418371..56ede10 100644 --- a/src/decagon_pytorch/data.py +++ b/src/decagon_pytorch/data.py @@ -1,3 +1,9 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + from collections import defaultdict from .decode import BilinearDecoder from .weights import init_glorot diff --git a/src/decagon_pytorch/decode.py b/src/decagon_pytorch/decode.py index fded66c..678eee2 100644 --- a/src/decagon_pytorch/decode.py +++ b/src/decagon_pytorch/decode.py @@ -1,3 +1,9 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + import torch from .weights import init_glorot from .dropout import dropout diff --git a/src/decagon_pytorch/dropout.py b/src/decagon_pytorch/dropout.py index 27196fe..f31b5dc 100644 --- a/src/decagon_pytorch/dropout.py +++ b/src/decagon_pytorch/dropout.py @@ -1,3 +1,9 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + import torch diff --git a/src/decagon_pytorch/layer/__init__.py b/src/decagon_pytorch/layer/__init__.py index c1ae674..dfb8b70 100644 --- a/src/decagon_pytorch/layer/__init__.py +++ b/src/decagon_pytorch/layer/__init__.py @@ -1,3 +1,9 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + # # This module implements a single layer of the Decagon # model. This is going to be already quite complex, as diff --git a/src/decagon_pytorch/layer/convolve.py b/src/decagon_pytorch/layer/convolve.py index 8229f36..baebd88 100644 --- a/src/decagon_pytorch/layer/convolve.py +++ b/src/decagon_pytorch/layer/convolve.py @@ -1,3 +1,9 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + from .layer import Layer import torch from ..convolve import DropoutGraphConvActivation diff --git a/src/decagon_pytorch/layer/decode.py b/src/decagon_pytorch/layer/decode.py index d0d548b..52a07b1 100644 --- a/src/decagon_pytorch/layer/decode.py +++ b/src/decagon_pytorch/layer/decode.py @@ -1,3 +1,9 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + from .layer import Layer import torch from ..data import Data diff --git a/src/decagon_pytorch/layer/input.py b/src/decagon_pytorch/layer/input.py index f794277..1b3a3f1 100644 --- a/src/decagon_pytorch/layer/input.py +++ b/src/decagon_pytorch/layer/input.py @@ -1,3 +1,9 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + from .layer import Layer import torch from typing import Union, \ diff --git a/src/decagon_pytorch/layer/layer.py b/src/decagon_pytorch/layer/layer.py index 91f705c..09a04d4 100644 --- a/src/decagon_pytorch/layer/layer.py +++ b/src/decagon_pytorch/layer/layer.py @@ -1,3 +1,9 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + import torch from typing import List, \ Union diff --git a/src/decagon_pytorch/model.py b/src/decagon_pytorch/model.py index 0e6380b..f99dc11 100644 --- a/src/decagon_pytorch/model.py +++ b/src/decagon_pytorch/model.py @@ -1,3 +1,9 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + class Model(object): def __init__(self, data): self.data = data diff --git a/src/decagon_pytorch/normalize.py b/src/decagon_pytorch/normalize.py index 577ad0d..7eb7150 100644 --- a/src/decagon_pytorch/normalize.py +++ b/src/decagon_pytorch/normalize.py @@ -1,3 +1,9 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + import numpy as np import scipy.sparse as sp diff --git a/src/decagon_pytorch/weights.py b/src/decagon_pytorch/weights.py index cfab885..2dcb7b4 100644 --- a/src/decagon_pytorch/weights.py +++ b/src/decagon_pytorch/weights.py @@ -1,3 +1,9 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + import torch import numpy as np -- 2.26.2 From b08ae2e160009b1310a2c51cb9ee4261ab4a58c9 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 1 Jun 2020 13:18:11 +0200 Subject: [PATCH 044/227] Test for an approach to pairwise (rather than cartesian product) link predition. --- src/decagon_pytorch/__init__.py | 2 +- tests/decagon_pytorch/test_decode_dims.py | 106 ++++++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 tests/decagon_pytorch/test_decode_dims.py diff --git a/src/decagon_pytorch/__init__.py b/src/decagon_pytorch/__init__.py index f95e8e8..8271fcc 100644 --- a/src/decagon_pytorch/__init__.py +++ b/src/decagon_pytorch/__init__.py @@ -6,4 +6,4 @@ from .weights import * from .convolve import * from .model import * -from .layer import * +from .layer.decode import * diff --git a/tests/decagon_pytorch/test_decode_dims.py b/tests/decagon_pytorch/test_decode_dims.py new file mode 100644 index 0000000..03d73cc --- /dev/null +++ b/tests/decagon_pytorch/test_decode_dims.py @@ -0,0 +1,106 @@ +from decagon_pytorch.decode import DEDICOMDecoder, \ + DistMultDecoder, \ + BilinearDecoder, \ + InnerProductDecoder +import torch + + +def _common(decoder_class): + decoder = decoder_class(input_dim=10, num_relation_types=1) + inputs = torch.rand((20, 10)) + pred = decoder(inputs, inputs) + + assert isinstance(pred, list) + assert len(pred) == 1 + + assert isinstance(pred[0], torch.Tensor) + assert pred[0].shape == (20, 20) + + + +def test_dedicom_decoder(): + _common(DEDICOMDecoder) + + +def test_dist_mult_decoder(): + _common(DistMultDecoder) + + +def test_bilinear_decoder(): + _common(BilinearDecoder) + + +def test_inner_product_decoder(): + _common(InnerProductDecoder) + + +def test_batch_matrix_multiplication(): + input_dim = 10 + inputs = torch.rand((20, 10)) + + decoder = DEDICOMDecoder(input_dim=input_dim, num_relation_types=1) + out_dec = decoder(inputs, inputs) + + relation = decoder.local_variation[0] + global_interaction = decoder.global_interaction + act = decoder.activation + relation = torch.diag(relation) + product1 = torch.mm(inputs, relation) + product2 = torch.mm(product1, global_interaction) + product3 = torch.mm(product2, relation) + rec = torch.mm(product3, torch.transpose(inputs, 0, 1)) + rec = act(rec) + + print('rec:', rec) + print('out_dec:', out_dec) + + assert torch.all(rec == out_dec[0]) + + +def test_single_prediction_01(): + input_dim = 10 + inputs = torch.rand((20, 10)) + + decoder = DEDICOMDecoder(input_dim=input_dim, num_relation_types=1) + dec_all = decoder(inputs, inputs) + dec_one = decoder(inputs[0:1], inputs[0:1]) + + assert torch.abs(dec_all[0][0, 0] - dec_one[0][0, 0]) < 0.000001 + + +def test_single_prediction_02(): + input_dim = 10 + inputs = torch.rand((20, 10)) + + decoder = DEDICOMDecoder(input_dim=input_dim, num_relation_types=1) + dec_all = decoder(inputs, inputs) + dec_one = decoder(inputs[0:1], inputs[1:2]) + + assert torch.abs(dec_all[0][0, 1] - dec_one[0][0, 0]) < 0.000001 + assert dec_one[0].shape == (1, 1) + + +def test_pairwise_prediction(): + n_nodes = 20 + input_dim = 10 + inputs_row = torch.rand((n_nodes, input_dim)) + inputs_col = torch.rand((n_nodes, input_dim)) + + decoder = DEDICOMDecoder(input_dim=input_dim, num_relation_types=1) + dec_all = decoder(inputs_row, inputs_col) + + relation = torch.diag(decoder.local_variation[0]) + global_interaction = decoder.global_interaction + act = decoder.activation + product1 = torch.mm(inputs_row, relation) + product2 = torch.mm(product1, global_interaction) + product3 = torch.mm(product2, relation) + rec = torch.bmm(product3.view(product3.shape[0], 1, product3.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + + assert rec.shape == (n_nodes, 1, 1) + + rec = torch.flatten(rec) + rec = act(rec) + + assert torch.all(torch.abs(rec - torch.diag(dec_all[0])) < 0.000001) -- 2.26.2 From c1689b4985c908340825f77eac7e690d005f33a8 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 1 Jun 2020 14:13:11 +0200 Subject: [PATCH 045/227] Split decoders into cartesian and pairwise. --- src/decagon_pytorch/data.py | 1 - src/decagon_pytorch/decode/__init__.py | 0 .../{decode.py => decode/cartesian.py} | 4 +- src/decagon_pytorch/decode/pairwise.py | 123 ++++++++++++++++++ src/decagon_pytorch/layer/decode.py | 2 +- .../layer/test_layer_decode.py | 2 +- tests/decagon_pytorch/test_decode.py | 10 +- tests/decagon_pytorch/test_decode_dims.py | 2 +- 8 files changed, 133 insertions(+), 11 deletions(-) create mode 100644 src/decagon_pytorch/decode/__init__.py rename src/decagon_pytorch/{decode.py => decode/cartesian.py} (95%) create mode 100644 src/decagon_pytorch/decode/pairwise.py diff --git a/src/decagon_pytorch/data.py b/src/decagon_pytorch/data.py index 56ede10..a375f75 100644 --- a/src/decagon_pytorch/data.py +++ b/src/decagon_pytorch/data.py @@ -5,7 +5,6 @@ from collections import defaultdict -from .decode import BilinearDecoder from .weights import init_glorot diff --git a/src/decagon_pytorch/decode/__init__.py b/src/decagon_pytorch/decode/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/decagon_pytorch/decode.py b/src/decagon_pytorch/decode/cartesian.py similarity index 95% rename from src/decagon_pytorch/decode.py rename to src/decagon_pytorch/decode/cartesian.py index 678eee2..910a8ce 100644 --- a/src/decagon_pytorch/decode.py +++ b/src/decagon_pytorch/decode/cartesian.py @@ -5,8 +5,8 @@ import torch -from .weights import init_glorot -from .dropout import dropout +from ..weights import init_glorot +from ..dropout import dropout class DEDICOMDecoder(torch.nn.Module): diff --git a/src/decagon_pytorch/decode/pairwise.py b/src/decagon_pytorch/decode/pairwise.py new file mode 100644 index 0000000..910a8ce --- /dev/null +++ b/src/decagon_pytorch/decode/pairwise.py @@ -0,0 +1,123 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import torch +from ..weights import init_glorot +from ..dropout import dropout + + +class DEDICOMDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, drop_prob=0., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.drop_prob = drop_prob + self.activation = activation + + self.global_interaction = init_glorot(input_dim, input_dim) + self.local_variation = [ + torch.flatten(init_glorot(input_dim, 1)) \ + for _ in range(num_relation_types) + ] + + def forward(self, inputs_row, inputs_col): + outputs = [] + for k in range(self.num_relation_types): + inputs_row = dropout(inputs_row, 1.-self.drop_prob) + inputs_col = dropout(inputs_col, 1.-self.drop_prob) + + relation = torch.diag(self.local_variation[k]) + + product1 = torch.mm(inputs_row, relation) + product2 = torch.mm(product1, self.global_interaction) + product3 = torch.mm(product2, relation) + rec = torch.mm(product3, torch.transpose(inputs_col, 0, 1)) + outputs.append(self.activation(rec)) + return outputs + + +class DistMultDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, drop_prob=0., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.drop_prob = drop_prob + self.activation = activation + + self.relation = [ + torch.flatten(init_glorot(input_dim, 1)) \ + for _ in range(num_relation_types) + ] + + def forward(self, inputs_row, inputs_col): + outputs = [] + for k in range(self.num_relation_types): + inputs_row = dropout(inputs_row, 1.-self.drop_prob) + inputs_col = dropout(inputs_col, 1.-self.drop_prob) + + relation = torch.diag(self.relation[k]) + + intermediate_product = torch.mm(inputs_row, relation) + rec = torch.mm(intermediate_product, torch.transpose(inputs_col, 0, 1)) + outputs.append(self.activation(rec)) + return outputs + + +class BilinearDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, drop_prob=0., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.drop_prob = drop_prob + self.activation = activation + + self.relation = [ + init_glorot(input_dim, input_dim) \ + for _ in range(num_relation_types) + ] + + def forward(self, inputs_row, inputs_col): + outputs = [] + for k in range(self.num_relation_types): + inputs_row = dropout(inputs_row, 1.-self.drop_prob) + inputs_col = dropout(inputs_col, 1.-self.drop_prob) + + intermediate_product = torch.mm(inputs_row, self.relation[k]) + rec = torch.mm(intermediate_product, torch.transpose(inputs_col, 0, 1)) + outputs.append(self.activation(rec)) + return outputs + + +class InnerProductDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, drop_prob=0., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.drop_prob = drop_prob + self.activation = activation + + + def forward(self, inputs_row, inputs_col): + outputs = [] + for k in range(self.num_relation_types): + inputs_row = dropout(inputs_row, 1.-self.drop_prob) + inputs_col = dropout(inputs_col, 1.-self.drop_prob) + + rec = torch.mm(inputs_row, torch.transpose(inputs_col, 0, 1)) + outputs.append(self.activation(rec)) + return outputs diff --git a/src/decagon_pytorch/layer/decode.py b/src/decagon_pytorch/layer/decode.py index 52a07b1..f354142 100644 --- a/src/decagon_pytorch/layer/decode.py +++ b/src/decagon_pytorch/layer/decode.py @@ -13,7 +13,7 @@ from typing import Type, \ Union, \ Dict, \ Tuple -from ..decode import DEDICOMDecoder +from ..decode.cartesian import DEDICOMDecoder class DecodeLayer(torch.nn.Module): diff --git a/tests/decagon_pytorch/layer/test_layer_decode.py b/tests/decagon_pytorch/layer/test_layer_decode.py index 57c554b..8c2e336 100644 --- a/tests/decagon_pytorch/layer/test_layer_decode.py +++ b/tests/decagon_pytorch/layer/test_layer_decode.py @@ -1,7 +1,7 @@ from decagon_pytorch.layer import OneHotInputLayer, \ DecagonLayer, \ DecodeLayer -from decagon_pytorch.decode import DEDICOMDecoder +from decagon_pytorch.decode.cartesian import DEDICOMDecoder from decagon_pytorch.data import Data import torch diff --git a/tests/decagon_pytorch/test_decode.py b/tests/decagon_pytorch/test_decode.py index 798fc50..de7d1a4 100644 --- a/tests/decagon_pytorch/test_decode.py +++ b/tests/decagon_pytorch/test_decode.py @@ -1,4 +1,4 @@ -import decagon_pytorch.decode +import decagon_pytorch.decode.cartesian import decagon.deep.layers import numpy as np import tensorflow as tf @@ -31,7 +31,7 @@ def _common(decoder_torch, decoder_tf): def test_dedicom_decoder(): - dedicom_torch = decagon_pytorch.decode.DEDICOMDecoder(input_dim=10, + dedicom_torch = decagon_pytorch.decode.cartesian.DEDICOMDecoder(input_dim=10, num_relation_types=7) dedicom_tf = decagon.deep.layers.DEDICOMDecoder(input_dim=10, num_types=7, edge_type=(0, 0)) @@ -46,7 +46,7 @@ def test_dedicom_decoder(): def test_dist_mult_decoder(): - distmult_torch = decagon_pytorch.decode.DistMultDecoder(input_dim=10, + distmult_torch = decagon_pytorch.decode.cartesian.DistMultDecoder(input_dim=10, num_relation_types=7) distmult_tf = decagon.deep.layers.DistMultDecoder(input_dim=10, num_types=7, edge_type=(0, 0)) @@ -59,7 +59,7 @@ def test_dist_mult_decoder(): def test_bilinear_decoder(): - bilinear_torch = decagon_pytorch.decode.BilinearDecoder(input_dim=10, + bilinear_torch = decagon_pytorch.decode.cartesian.BilinearDecoder(input_dim=10, num_relation_types=7) bilinear_tf = decagon.deep.layers.BilinearDecoder(input_dim=10, num_types=7, edge_type=(0, 0)) @@ -72,7 +72,7 @@ def test_bilinear_decoder(): def test_inner_product_decoder(): - inner_torch = decagon_pytorch.decode.InnerProductDecoder(input_dim=10, + inner_torch = decagon_pytorch.decode.cartesian.InnerProductDecoder(input_dim=10, num_relation_types=7) inner_tf = decagon.deep.layers.InnerProductDecoder(input_dim=10, num_types=7, edge_type=(0, 0)) diff --git a/tests/decagon_pytorch/test_decode_dims.py b/tests/decagon_pytorch/test_decode_dims.py index 03d73cc..2c3a144 100644 --- a/tests/decagon_pytorch/test_decode_dims.py +++ b/tests/decagon_pytorch/test_decode_dims.py @@ -1,4 +1,4 @@ -from decagon_pytorch.decode import DEDICOMDecoder, \ +from decagon_pytorch.decode.cartesian import DEDICOMDecoder, \ DistMultDecoder, \ BilinearDecoder, \ InnerProductDecoder -- 2.26.2 From 39aa1b42110fd59470ef55de35f3683be58427e6 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 1 Jun 2020 14:30:13 +0200 Subject: [PATCH 046/227] Implement and test pairwise decoders. --- src/decagon_pytorch/decode/pairwise.py | 16 +++-- tests/decagon_pytorch/test_decode_pairwise.py | 59 +++++++++++++++++++ 2 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 tests/decagon_pytorch/test_decode_pairwise.py diff --git a/src/decagon_pytorch/decode/pairwise.py b/src/decagon_pytorch/decode/pairwise.py index 910a8ce..93ff68c 100644 --- a/src/decagon_pytorch/decode/pairwise.py +++ b/src/decagon_pytorch/decode/pairwise.py @@ -37,7 +37,9 @@ class DEDICOMDecoder(torch.nn.Module): product1 = torch.mm(inputs_row, relation) product2 = torch.mm(product1, self.global_interaction) product3 = torch.mm(product2, relation) - rec = torch.mm(product3, torch.transpose(inputs_col, 0, 1)) + rec = torch.bmm(product3.view(product3.shape[0], 1, product3.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) outputs.append(self.activation(rec)) return outputs @@ -67,7 +69,9 @@ class DistMultDecoder(torch.nn.Module): relation = torch.diag(self.relation[k]) intermediate_product = torch.mm(inputs_row, relation) - rec = torch.mm(intermediate_product, torch.transpose(inputs_col, 0, 1)) + rec = torch.bmm(intermediate_product.view(intermediate_product.shape[0], 1, intermediate_product.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) outputs.append(self.activation(rec)) return outputs @@ -95,7 +99,9 @@ class BilinearDecoder(torch.nn.Module): inputs_col = dropout(inputs_col, 1.-self.drop_prob) intermediate_product = torch.mm(inputs_row, self.relation[k]) - rec = torch.mm(intermediate_product, torch.transpose(inputs_col, 0, 1)) + rec = torch.bmm(intermediate_product.view(intermediate_product.shape[0], 1, intermediate_product.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) outputs.append(self.activation(rec)) return outputs @@ -118,6 +124,8 @@ class InnerProductDecoder(torch.nn.Module): inputs_row = dropout(inputs_row, 1.-self.drop_prob) inputs_col = dropout(inputs_col, 1.-self.drop_prob) - rec = torch.mm(inputs_row, torch.transpose(inputs_col, 0, 1)) + rec = torch.bmm(inputs_row.view(inputs_row.shape[0], 1, inputs_row.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) outputs.append(self.activation(rec)) return outputs diff --git a/tests/decagon_pytorch/test_decode_pairwise.py b/tests/decagon_pytorch/test_decode_pairwise.py new file mode 100644 index 0000000..9b47eef --- /dev/null +++ b/tests/decagon_pytorch/test_decode_pairwise.py @@ -0,0 +1,59 @@ +import decagon_pytorch.decode.cartesian as cart +import decagon_pytorch.decode.pairwise as pair +import torch + + +def _common(cart_class, pair_class): + input_dim = 10 + n_nodes = 20 + num_relation_types = 7 + + inputs_row = torch.rand((n_nodes, input_dim)) + inputs_col = torch.rand((n_nodes, input_dim)) + + cart_dec = cart_class(input_dim=input_dim, + num_relation_types=num_relation_types) + pair_dec = pair_class(input_dim=input_dim, + num_relation_types=num_relation_types) + + if isinstance(cart_dec, cart.DEDICOMDecoder): + pair_dec.global_interaction = cart_dec.global_interaction + pair_dec.local_variation = cart_dec.local_variation + elif isinstance(cart_dec, cart.InnerProductDecoder): + pass + else: + pair_dec.relation = cart_dec.relation + + cart_pred = cart_dec(inputs_row, inputs_col) + pair_pred = pair_dec(inputs_row, inputs_col) + + assert isinstance(cart_pred, list) + assert isinstance(pair_pred, list) + + assert len(cart_pred) == len(pair_pred) + assert len(cart_pred) == num_relation_types + + for i in range(num_relation_types): + assert isinstance(cart_pred[i], torch.Tensor) + assert isinstance(pair_pred[i], torch.Tensor) + + assert cart_pred[i].shape == (n_nodes, n_nodes) + assert pair_pred[i].shape == (n_nodes,) + + assert torch.all(torch.abs(pair_pred[i] - torch.diag(cart_pred[i])) < 0.000001) + + +def test_dedicom_decoder(): + _common(cart.DEDICOMDecoder, pair.DEDICOMDecoder) + + +def test_dist_mult_decoder(): + _common(cart.DistMultDecoder, pair.DistMultDecoder) + + +def test_bilinear_decoder(): + _common(cart.BilinearDecoder, pair.BilinearDecoder) + + +def test_inner_product_decoder(): + _common(cart.InnerProductDecoder, pair.InnerProductDecoder) -- 2.26.2 From f2121bef539c0ac9c23e72e908bd417064f966a8 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 2 Jun 2020 11:50:51 +0200 Subject: [PATCH 047/227] Add fixed_unigram_candidate_sampler(). --- src/decagon_pytorch/sampling.py | 38 +++++++ tests/decagon_pytorch/test_unigram.py | 141 ++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 src/decagon_pytorch/sampling.py create mode 100644 tests/decagon_pytorch/test_unigram.py diff --git a/src/decagon_pytorch/sampling.py b/src/decagon_pytorch/sampling.py new file mode 100644 index 0000000..a029626 --- /dev/null +++ b/src/decagon_pytorch/sampling.py @@ -0,0 +1,38 @@ +import numpy as np +import torch +import torch.utils.data +from typing import List, \ + Union + + +def fixed_unigram_candidate_sampler( + true_classes: Union[np.array, torch.Tensor], + num_true: int, + num_samples: int, + range_max: int, + unigrams: List[Union[int, float]], + distortion: float = 1.): + + if isinstance(true_classes, torch.Tensor): + true_classes = true_classes.detach().cpu().numpy() + if true_classes.shape != (num_samples, num_true): + raise ValueError('true_classes must be a 2D matrix with shape (num_samples, num_true)') + unigrams = np.array(unigrams) + if distortion != 1.: + unigrams = unigrams.astype(np.float64) ** distortion + # print('unigrams:', unigrams) + indices = np.arange(num_samples) + result = np.zeros(num_samples, dtype=np.int64) + while len(indices) > 0: + # print('len(indices):', len(indices)) + sampler = torch.utils.data.WeightedRandomSampler(unigrams, len(indices)) + candidates = np.array(list(sampler)) + candidates = np.reshape(candidates, (len(indices), 1)) + # print('candidates:', candidates) + # print('true_classes:', true_classes[indices, :]) + result[indices] = candidates.T + mask = (candidates == true_classes[indices, :]) + mask = mask.sum(1).astype(np.bool) + # print('mask:', mask) + indices = indices[mask] + return result diff --git a/tests/decagon_pytorch/test_unigram.py b/tests/decagon_pytorch/test_unigram.py new file mode 100644 index 0000000..c08c1bd --- /dev/null +++ b/tests/decagon_pytorch/test_unigram.py @@ -0,0 +1,141 @@ +import tensorflow as tf +import numpy as np +from collections import defaultdict +import torch +import torch.utils.data +from typing import List, \ + Union +import decagon_pytorch.sampling + + +def test_unigram_01(): + range_max = 7 + distortion = 0.75 + batch_size = 500 + unigrams = [ 1, 3, 2, 1, 2, 1, 3] + num_true = 1 + + true_classes = np.zeros((batch_size, num_true), dtype=np.int64) + for i in range(batch_size): + true_classes[i, 0] = i % range_max + true_classes = tf.convert_to_tensor(true_classes) + + neg_samples, _, _ = tf.nn.fixed_unigram_candidate_sampler( + true_classes=true_classes, + num_true=num_true, + num_sampled=batch_size, + unique=False, + range_max=range_max, + distortion=distortion, + unigrams=unigrams) + + assert neg_samples.shape == (batch_size,) + + for i in range(batch_size): + assert neg_samples[i] != true_classes[i, 0] + + counts = defaultdict(int) + with tf.Session() as sess: + neg_samples = neg_samples.eval() + for x in neg_samples: + counts[x] += 1 + + print('counts:', counts) + + assert counts[0] < counts[1] and \ + counts[0] < counts[2] and \ + counts[0] < counts[4] and \ + counts[0] < counts[6] + + assert counts[2] < counts[1] and \ + counts[0] < counts[6] + + assert counts[3] < counts[1] and \ + counts[3] < counts[2] and \ + counts[3] < counts[4] and \ + counts[3] < counts[6] + + assert counts[4] < counts[1] and \ + counts[4] < counts[6] + + assert counts[5] < counts[1] and \ + counts[5] < counts[2] and \ + counts[5] < counts[4] and \ + counts[5] < counts[6] + + +def test_unigram_02(): + range_max = 7 + distortion = 0.75 + batch_size = 500 + unigrams = [ 1, 3, 2, 1, 2, 1, 3] + num_true = 1 + + true_classes = np.zeros((batch_size, num_true), dtype=np.int64) + for i in range(batch_size): + true_classes[i, 0] = i % range_max + true_classes = torch.tensor(true_classes) + + neg_samples = decagon_pytorch.sampling.fixed_unigram_candidate_sampler( + true_classes=true_classes, + num_true=num_true, + num_samples=batch_size, + range_max=range_max, + distortion=distortion, + unigrams=unigrams) + + assert neg_samples.shape == (batch_size,) + + for i in range(batch_size): + assert neg_samples[i] != true_classes[i, 0] + + counts = defaultdict(int) + for x in neg_samples: + counts[x] += 1 + + print('counts:', counts) + + assert counts[0] < counts[1] and \ + counts[0] < counts[2] and \ + counts[0] < counts[4] and \ + counts[0] < counts[6] + + assert counts[2] < counts[1] and \ + counts[0] < counts[6] + + assert counts[3] < counts[1] and \ + counts[3] < counts[2] and \ + counts[3] < counts[4] and \ + counts[3] < counts[6] + + assert counts[4] < counts[1] and \ + counts[4] < counts[6] + + assert counts[5] < counts[1] and \ + counts[5] < counts[2] and \ + counts[5] < counts[4] and \ + counts[5] < counts[6] + + +def test_unigram_03(): + range_max = 7 + distortion = 0.75 + batch_size = 500 + unigrams = [ 1, 3, 2, 1, 2, 1, 3] + num_true = 1 + + true_classes = np.zeros((batch_size, num_true), dtype=np.int64) + for i in range(batch_size): + true_classes[i, 0] = i % range_max + true_classes_tf = tf.convert_to_tensor(true_classes) + + neg_samples, _, _ = tf.nn.fixed_unigram_candidate_sampler( + true_classes=true_classes_tf, + num_true=num_true, + num_sampled=batch_size, + unique=False, + range_max=range_max, + distortion=distortion, + unigrams=unigrams) + + true_classes_torch = torch.tensor(true_classes) -- 2.26.2 From 7ab97e2fb65330c3cd9282d59e47f727ed37e4f2 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 2 Jun 2020 12:08:34 +0200 Subject: [PATCH 048/227] Add test_unigram_03(). --- src/decagon_pytorch/sampling.py | 4 +- .../{test_unigram.py => test_sampling.py} | 57 ++++++++++++++----- 2 files changed, 45 insertions(+), 16 deletions(-) rename tests/decagon_pytorch/{test_unigram.py => test_sampling.py} (68%) diff --git a/src/decagon_pytorch/sampling.py b/src/decagon_pytorch/sampling.py index a029626..c2357a3 100644 --- a/src/decagon_pytorch/sampling.py +++ b/src/decagon_pytorch/sampling.py @@ -7,15 +7,13 @@ from typing import List, \ def fixed_unigram_candidate_sampler( true_classes: Union[np.array, torch.Tensor], - num_true: int, num_samples: int, - range_max: int, unigrams: List[Union[int, float]], distortion: float = 1.): if isinstance(true_classes, torch.Tensor): true_classes = true_classes.detach().cpu().numpy() - if true_classes.shape != (num_samples, num_true): + if true_classes.shape[0] != num_samples: raise ValueError('true_classes must be a 2D matrix with shape (num_samples, num_true)') unigrams = np.array(unigrams) if distortion != 1.: diff --git a/tests/decagon_pytorch/test_unigram.py b/tests/decagon_pytorch/test_sampling.py similarity index 68% rename from tests/decagon_pytorch/test_unigram.py rename to tests/decagon_pytorch/test_sampling.py index c08c1bd..028d5b5 100644 --- a/tests/decagon_pytorch/test_unigram.py +++ b/tests/decagon_pytorch/test_sampling.py @@ -6,6 +6,7 @@ import torch.utils.data from typing import List, \ Union import decagon_pytorch.sampling +import scipy.stats def test_unigram_01(): @@ -78,9 +79,7 @@ def test_unigram_02(): neg_samples = decagon_pytorch.sampling.fixed_unigram_candidate_sampler( true_classes=true_classes, - num_true=num_true, num_samples=batch_size, - range_max=range_max, distortion=distortion, unigrams=unigrams) @@ -120,22 +119,54 @@ def test_unigram_02(): def test_unigram_03(): range_max = 7 distortion = 0.75 - batch_size = 500 + batch_size = 25 unigrams = [ 1, 3, 2, 1, 2, 1, 3] num_true = 1 true_classes = np.zeros((batch_size, num_true), dtype=np.int64) for i in range(batch_size): true_classes[i, 0] = i % range_max - true_classes_tf = tf.convert_to_tensor(true_classes) - - neg_samples, _, _ = tf.nn.fixed_unigram_candidate_sampler( - true_classes=true_classes_tf, - num_true=num_true, - num_sampled=batch_size, - unique=False, - range_max=range_max, - distortion=distortion, - unigrams=unigrams) + true_classes_tf = tf.convert_to_tensor(true_classes) true_classes_torch = torch.tensor(true_classes) + + counts_tf = defaultdict(list) + counts_torch = defaultdict(list) + + for i in range(100): + neg_samples, _, _ = tf.nn.fixed_unigram_candidate_sampler( + true_classes=true_classes_tf, + num_true=num_true, + num_sampled=batch_size, + unique=False, + range_max=range_max, + distortion=distortion, + unigrams=unigrams) + + counts = defaultdict(int) + with tf.Session() as sess: + neg_samples = neg_samples.eval() + for x in neg_samples: + counts[x] += 1 + for k, v in counts.items(): + counts_tf[k].append(v) + + neg_samples = decagon_pytorch.sampling.fixed_unigram_candidate_sampler( + true_classes=true_classes, + num_samples=batch_size, + distortion=distortion, + unigrams=unigrams) + + counts = defaultdict(int) + for x in neg_samples: + counts[x] += 1 + for k, v in counts.items(): + counts_torch[k].append(v) + + for i in range(range_max): + print('counts_tf[%d]:' % i, counts_tf[i]) + print('counts_torch[%d]:' % i, counts_torch[i]) + + for i in range(range_max): + statistic, pvalue = scipy.stats.ttest_ind(counts_tf[i], counts_torch[i]) + assert pvalue * range_max > .05 -- 2.26.2 From a74884daddfa69d3c6dd79379beebc4b4070e6de Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 2 Jun 2020 16:28:35 +0200 Subject: [PATCH 049/227] Add train_val_test_split_adj_mat(). --- src/decagon_pytorch/splits.py | 26 ++++++++ tests/decagon_pytorch/test_splits.py | 95 ++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 src/decagon_pytorch/splits.py create mode 100644 tests/decagon_pytorch/test_splits.py diff --git a/src/decagon_pytorch/splits.py b/src/decagon_pytorch/splits.py new file mode 100644 index 0000000..e0394cc --- /dev/null +++ b/src/decagon_pytorch/splits.py @@ -0,0 +1,26 @@ +import torch + + +def train_val_test_split_adj_mat(adj_mat, train_ratio, val_ratio, test_ratio): + if train_ratio + val_ratio + test_ratio != 1.0: + raise ValueError('Train, validation and test ratios must add up to 1') + + edges = torch.nonzero(adj_mat) + order = torch.randperm(len(edges)) + edges = edges[order, :] + n = round(len(edges) * train_ratio) + edges_train = edges[:n] + n_1 = round(len(edges) * (train_ratio + val_ratio)) + edges_val = edges[n:n_1] + edges_test = edges[n_1:] + + adj_mat_train = torch.zeros_like(adj_mat) + adj_mat_train[edges_train[:, 0], edges_train[:, 1]] = 1 + + adj_mat_val = torch.zeros_like(adj_mat) + adj_mat_val[edges_val[:, 0], edges_val[:, 1]] = 1 + + adj_mat_test = torch.zeros_like(adj_mat) + adj_mat_test[edges_test[:, 0], edges_test[:, 1]] = 1 + + return adj_mat_train, adj_mat_val, adj_mat_test diff --git a/tests/decagon_pytorch/test_splits.py b/tests/decagon_pytorch/test_splits.py new file mode 100644 index 0000000..79482fb --- /dev/null +++ b/tests/decagon_pytorch/test_splits.py @@ -0,0 +1,95 @@ +from decagon_pytorch.data import Data +import torch +from decagon_pytorch.splits import train_val_test_split_adj_mat +import pytest + + +def _gen_adj_mat(n_rows, n_cols): + res = torch.rand((n_rows, n_cols)).round() + if n_rows == n_cols: + res -= torch.diag(torch.diag(res)) + a, b = torch.triu_indices(n_rows, n_cols) + res[a, b] = res.transpose(0, 1)[a, b] + return res + + +def train_val_test_split_1(data, train_ratio=0.8, + val_ratio=0.1, test_ratio=0.1): + + if train_ratio + val_ratio + test_ratio != 1.0: + raise ValueError('Train, validation and test ratios must add up to 1') + + d_train = Data() + d_val = Data() + d_test = Data() + + for (node_type_row, node_type_col), rels in data.relation_types.items(): + for r in rels: + adj_train, adj_val, adj_test = train_val_test_split_adj_mat(r.adjacency_matrix) + d_train.add_relation_type(r.name, node_type_row, node_type_col, adj_train) + d_val.add_relation_type(r.name, node_type_row, node_type_col, adj_train + adj_val) + + +def train_val_test_split_2(data, train_ratio, val_ratio, test_ratio): + if train_ratio + val_ratio + test_ratio != 1.0: + raise ValueError('Train, validation and test ratios must add up to 1') + for (node_type_row, node_type_col), rels in data.relation_types.items(): + for r in rels: + adj_mat = r.adjacency_matrix + edges = torch.nonzero(adj_mat) + order = torch.randperm(len(edges)) + edges = edges[order, :] + n = round(len(edges) * train_ratio) + edges_train = edges[:n] + n_1 = round(len(edges) * (train_ratio + val_ratio)) + edges_val = edges[n:n_1] + edges_test = edges[n_1:] + if len(edges_train) * len(edges_val) * len(edges_test) == 0: + raise ValueError('Not enough edges to split into train/val/test sets for: ' + r.name) + + +def test_train_val_test_split_adj_mat(): + adj_mat = _gen_adj_mat(50, 100) + adj_mat_train, adj_mat_val, adj_mat_test = \ + train_val_test_split_adj_mat(adj_mat, train_ratio=0.8, + val_ratio=0.1, test_ratio=0.1) + + assert adj_mat.shape == adj_mat_train.shape == \ + adj_mat_val.shape == adj_mat_test.shape + + edges_train = torch.nonzero(adj_mat_train) + edges_val = torch.nonzero(adj_mat_val) + edges_test = torch.nonzero(adj_mat_test) + + edges_train = set(map(tuple, edges_train.tolist())) + edges_val = set(map(tuple, edges_val.tolist())) + edges_test = set(map(tuple, edges_test.tolist())) + + assert edges_train.intersection(edges_val) == set() + assert edges_train.intersection(edges_test) == set() + assert edges_test.intersection(edges_val) == set() + + assert torch.all(adj_mat_train + adj_mat_val + adj_mat_test == adj_mat) + + # assert torch.all((edges_train != edges_val).sum(1).to(torch.bool)) + # assert torch.all((edges_train != edges_test).sum(1).to(torch.bool)) + # assert torch.all((edges_val != edges_test).sum(1).to(torch.bool)) + + +@pytest.mark.skip +def test_splits_01(): + d = Data() + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + d.add_relation_type('Interaction', 0, 0, + _gen_adj_mat(1000, 1000)) + d.add_relation_type('Target', 1, 0, + _gen_adj_mat(100, 1000)) + d.add_relation_type('Side Effect: Insomnia', 1, 1, + _gen_adj_mat(100, 100)) + d.add_relation_type('Side Effect: Incontinence', 1, 1, + _gen_adj_mat(100, 100)) + d.add_relation_type('Side Effect: Abdominal pain', 1, 1, + _gen_adj_mat(100, 100)) + + d_train, d_val, d_test = train_val_test_split(d, 0.8, 0.1, 0.1) -- 2.26.2 From fab210448d268d8833ace924a95c80b918924fbb Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 3 Jun 2020 16:45:03 +0200 Subject: [PATCH 050/227] Correct support for assymetric relations in Data. --- src/decagon_pytorch/data.py | 48 ++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/src/decagon_pytorch/data.py b/src/decagon_pytorch/data.py index a375f75..762b1a6 100644 --- a/src/decagon_pytorch/data.py +++ b/src/decagon_pytorch/data.py @@ -16,11 +16,16 @@ class NodeType(object): class RelationType(object): def __init__(self, name, node_type_row, node_type_column, - adjacency_matrix): + adjacency_matrix, adjacency_matrix_transposed): + + if adjacency_matrix_transposed.shape != adjacency_matrix.transpose(0, 1).shape: + raise ValueError('adjacency_matrix_transposed has incorrect shape') + self.name = name self.node_type_row = node_type_row self.node_type_column = node_type_column self.adjacency_matrix = adjacency_matrix + self.adjacency_matrix_transposed = adjacency_matrix_transposed def get_adjacency_matrix(node_type_row, node_type_column): if self.node_type_row == node_type_row and \ @@ -29,7 +34,10 @@ class RelationType(object): elif self.node_type_row == node_type_column and \ self.node_type_column == node_type_row: - return self.adjacency_matrix.transpose(0, 1) + if self.adjacency_matrix_transposed: + return self.adjacency_matrix_transposed + else: + return self.adjacency_matrix.transpose(0, 1) else: raise ValueError('Specified row/column types do not correspond to this relation') @@ -39,37 +47,27 @@ class Data(object): def __init__(self): self.node_types = [] self.relation_types = defaultdict(list) - # self.decoder_types = defaultdict(lambda: BilinearDecoder) - # self.latent_node = [] def add_node_type(self, name, count): # , latent_length): self.node_types.append(NodeType(name, count)) - # self.latent_node.append(init_glorot(count, latent_length)) - def add_relation_type(self, name, node_type_row, node_type_column, adjacency_matrix): + def add_relation_type(self, name, node_type_row, node_type_column, adjacency_matrix, adjacency_matrix_transposed=None): n = len(self.node_types) if node_type_row >= n or node_type_column >= n: raise ValueError('Node type index out of bounds, add node type first') key = (node_type_row, node_type_column) if adjacency_matrix is not None and not adjacency_matrix.is_sparse: adjacency_matrix = adjacency_matrix.to_sparse() - self.relation_types[key].append(RelationType(name, node_type_row, node_type_column, adjacency_matrix)) - # _ = self.decoder_types[(node_type_row, node_type_column)] - - #def set_decoder_type(self, node_type_row, node_type_column, decoder_class): - # if (node_type_row, node_type_column) not in self.decoder_types: - # raise ValueError('Relation type not found, add relation first') - # self.decoder_types[(node_type_row, node_type_column)] = decoder_class + self.relation_types[key].append(RelationType(name, node_type_row, node_type_column, adjacency_matrix, adjacency_matrix_transposed)) def get_adjacency_matrices(self, node_type_row, node_type_column): - # rels = list(filter(lambda a: a[0] == node_type_row and a[1] == node_type_column), self.relation_types) - key = (node_type_row, node_type_column) - if key not in self.relation_types: - raise ValueError('Relation type not found') - rels = self.relation_types[key] - rels = list(map(lambda a: a.adjacency_matrix, rels)) - return rels - + res = [] + for (i, j), rels in self.relation_types.items(): + if node_type_row not in [i, j] and node_type_column not in [i, j]: + continue + for r in rels: + res.append(r.get_adjacency_matrix(node_type_row, node_type_column)) + return res def __repr__(self): n = len(self.node_types) @@ -91,13 +89,7 @@ class Data(object): if key not in self.relation_types: continue rels = self.relation_types[key] - # rels = list(filter(lambda a: a[0] == i and a[1] == j, self.relation_types)) - #if len(rels) == 0: - # continue - # dir = '<->' if i == j else '->' - dir = '--' - s += ' - ' + self.node_types[i].name + ' ' + dir + ' ' + self.node_types[j].name + ':\n' - #' (' + self.decoder_types[(i, j)].__name__ + '):\n' + s += ' - ' + self.node_types[i].name + ' -- ' + self.node_types[j].name + ':\n' for r in rels: s += ' - ' + r.name + '\n' return s.strip() -- 2.26.2 From 3a0c8aae52618de91bab0296e3c45eb692d62624 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 5 Jun 2020 00:40:45 +0200 Subject: [PATCH 051/227] Add train-val-test-diagram. --- docs/train-val-test-diagram.svg | 1445 +++++++++++++++++++++++++++++++ 1 file changed, 1445 insertions(+) create mode 100644 docs/train-val-test-diagram.svg diff --git a/docs/train-val-test-diagram.svg b/docs/train-val-test-diagram.svg new file mode 100644 index 0000000..90428c0 --- /dev/null +++ b/docs/train-val-test-diagram.svg @@ -0,0 +1,1445 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + Atrain + + + + + + + + Ltrain+ + + + + + + Ltrain- + + + + + + + + Ltest + + + + + + Atrain + + + + + + + + + + 1101001001 + + + 1101001001 + + + 1101001001 + + + 1101001001 + + + + + + + + + + + + + 1101001001 + + + 1101001001 + + + 1101001001 + + + 1101001001 + + + ... + + + + + + + + + + 1101001001 + + + 1101001001 + + + 1101001001 + + + 1101001001 + + + + + + 1101001001 + + + 1101001001 + + + 1101001001 + + + 1101001001 + + + + + 1101001001 + + + 1101001001 + + + 1101001001 + + + 1101001001 + + + + + + + + + + + + + Ltrain+ + + + + + + Ltrain- + 01010100 cross-entropy training + + + + + + + + Lval+ + + + + + + Lval- + 01010100 cross-entropy + + + gradient + stop criterion + + + + + + + Ltest+ + + + + + + Ltest- + 01010100 METRICS + + + + -- 2.26.2 From 28272e3d296e86570307f90251ef25417e62a6f5 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 5 Jun 2020 14:46:32 +0200 Subject: [PATCH 052/227] Add AdjListRelationType, AdjListData. --- src/decagon_pytorch/data/__init__.py | 2 + src/decagon_pytorch/data/list.py | 68 +++++++++++++++++++ .../{data.py => data/matrix.py} | 5 +- tests/decagon_pytorch/test_data_list.py | 67 ++++++++++++++++++ .../{test_data.py => test_data_matrix.py} | 0 5 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 src/decagon_pytorch/data/__init__.py create mode 100644 src/decagon_pytorch/data/list.py rename src/decagon_pytorch/{data.py => data/matrix.py} (92%) create mode 100644 tests/decagon_pytorch/test_data_list.py rename tests/decagon_pytorch/{test_data.py => test_data_matrix.py} (100%) diff --git a/src/decagon_pytorch/data/__init__.py b/src/decagon_pytorch/data/__init__.py new file mode 100644 index 0000000..5820dbb --- /dev/null +++ b/src/decagon_pytorch/data/__init__.py @@ -0,0 +1,2 @@ +from .matrix import * +from .list import * diff --git a/src/decagon_pytorch/data/list.py b/src/decagon_pytorch/data/list.py new file mode 100644 index 0000000..ca022cb --- /dev/null +++ b/src/decagon_pytorch/data/list.py @@ -0,0 +1,68 @@ +from .matrix import NodeType +import torch +from collections import defaultdict + + +class AdjListRelationType(object): + def __init__(self, name, node_type_row, node_type_column, + adjacency_list, adjacency_list_transposed=None): + + #if adjacency_matrix_transposed is not None and \ + # adjacency_matrix_transposed.shape != adjacency_matrix.transpose(0, 1).shape: + # raise ValueError('adjacency_matrix_transposed has incorrect shape') + + self.name = name + self.node_type_row = node_type_row + self.node_type_column = node_type_column + self.adjacency_list = adjacency_list + self.adjacency_list_transposed = adjacency_list_transposed + + def get_adjacency_list(self, node_type_row, node_type_column): + if self.node_type_row == node_type_row and \ + self.node_type_column == node_type_column: + return self.adjacency_list + + elif self.node_type_row == node_type_column and \ + self.node_type_column == node_type_row: + if self.adjacency_list_transposed is not None: + return self.adjacency_list_transposed + else: + return torch.index_select(self.adjacency_list, 1, + torch.LongTensor([1, 0])) + + else: + raise ValueError('Specified row/column types do not correspond to this relation') + + +def _verify_adjacency_list(adjacency_list, node_count_row, node_count_col): + assert isinstance(adjacency_list, torch.Tensor) + assert len(adjacency_list.shape) == 2 + assert torch.all(adjacency_list[:, 0] >= 0) + assert torch.all(adjacency_list[:, 0] < node_count_row) + assert torch.all(adjacency_list[:, 1] >= 0) + assert torch.all(adjacency_list[:, 1] < node_count_col) + + +class AdjListData(object): + def __init__(self): + self.node_types = [] + self.relation_types = defaultdict(list) + + def add_node_type(self, name, count): # , latent_length): + self.node_types.append(NodeType(name, count)) + + def add_relation_type(self, name, node_type_row, node_type_col, adjacency_list, adjacency_list_transposed=None): + assert node_type_row >= 0 and node_type_row < len(self.node_types) + assert node_type_col >= 0 and node_type_col < len(self.node_types) + + node_count_row = self.node_types[node_type_row].count + node_count_col = self.node_types[node_type_col].count + + _verify_adjacency_list(adjacency_list, node_count_row, node_count_col) + if adjacency_list_transposed is not None: + _verify_adjacency_list(adjacency_list_transposed, + node_count_col, node_count_row) + + self.relation_types[node_type_row, node_type_col].append( + AdjListRelationType(name, node_type_row, node_type_col, + adjacency_list, adjacency_list_transposed)) diff --git a/src/decagon_pytorch/data.py b/src/decagon_pytorch/data/matrix.py similarity index 92% rename from src/decagon_pytorch/data.py rename to src/decagon_pytorch/data/matrix.py index 762b1a6..cd4b110 100644 --- a/src/decagon_pytorch/data.py +++ b/src/decagon_pytorch/data/matrix.py @@ -5,7 +5,7 @@ from collections import defaultdict -from .weights import init_glorot +from ..weights import init_glorot class NodeType(object): @@ -18,7 +18,8 @@ class RelationType(object): def __init__(self, name, node_type_row, node_type_column, adjacency_matrix, adjacency_matrix_transposed): - if adjacency_matrix_transposed.shape != adjacency_matrix.transpose(0, 1).shape: + if adjacency_matrix_transposed is not None and \ + adjacency_matrix_transposed.shape != adjacency_matrix.transpose(0, 1).shape: raise ValueError('adjacency_matrix_transposed has incorrect shape') self.name = name diff --git a/tests/decagon_pytorch/test_data_list.py b/tests/decagon_pytorch/test_data_list.py new file mode 100644 index 0000000..a9cdb51 --- /dev/null +++ b/tests/decagon_pytorch/test_data_list.py @@ -0,0 +1,67 @@ +from decagon_pytorch.data import AdjListData, \ + AdjListRelationType +import torch +import pytest + + +def _get_list(): + lst = torch.tensor([ + [0, 1], + [0, 3], + [0, 5], + [0, 7] + ]) + return lst + + +def test_adj_list_relation_type_01(): + lst = _get_list() + rel = AdjListRelationType('Test', 0, 0, lst) + assert torch.all(rel.get_adjacency_list(0, 0) == lst) + + +def test_adj_list_relation_type_02(): + lst = _get_list() + rel = AdjListRelationType('Test', 0, 1, lst) + assert torch.all(rel.get_adjacency_list(0, 1) == lst) + lst_2 = torch.tensor([ + [1, 0], + [3, 0], + [5, 0], + [7, 0] + ]) + assert torch.all(rel.get_adjacency_list(1, 0) == lst_2) + + +def test_adj_list_relation_type_03(): + lst = _get_list() + lst_2 = torch.tensor([ + [2, 0], + [4, 0], + [6, 0], + [8, 0] + ]) + rel = AdjListRelationType('Test', 0, 1, lst, lst_2) + assert torch.all(rel.get_adjacency_list(0, 1) == lst) + assert torch.all(rel.get_adjacency_list(1, 0) == lst_2) + + +def test_adj_list_data_01(): + lst = _get_list() + d = AdjListData() + with pytest.raises(AssertionError): + d.add_relation_type('Test', 0, 1, lst) + d.add_node_type('Drugs', 5) + with pytest.raises(AssertionError): + d.add_relation_type('Test', 0, 0, lst) + d = AdjListData() + d.add_node_type('Drugs', 8) + d.add_relation_type('Test', 0, 0, lst) + + +def test_adj_list_data_02(): + lst = _get_list() + d = AdjListData() + d.add_node_type('Drugs', 10) + d.add_node_type('Proteins', 10) + d.add_relation_type('Target', 0, 1, lst) diff --git a/tests/decagon_pytorch/test_data.py b/tests/decagon_pytorch/test_data_matrix.py similarity index 100% rename from tests/decagon_pytorch/test_data.py rename to tests/decagon_pytorch/test_data_matrix.py -- 2.26.2 From e201608cafc191bc2a2aa772f0a66b41dcb92d08 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 5 Jun 2020 15:07:58 +0200 Subject: [PATCH 053/227] Add trainprep. --- src/decagon_pytorch/data/trainprep.py | 2 ++ src/decagon_pytorch/splits.py | 38 +++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 src/decagon_pytorch/data/trainprep.py diff --git a/src/decagon_pytorch/data/trainprep.py b/src/decagon_pytorch/data/trainprep.py new file mode 100644 index 0000000..18373ea --- /dev/null +++ b/src/decagon_pytorch/data/trainprep.py @@ -0,0 +1,2 @@ +def trainprep(data): + pass diff --git a/src/decagon_pytorch/splits.py b/src/decagon_pytorch/splits.py index e0394cc..9d219b6 100644 --- a/src/decagon_pytorch/splits.py +++ b/src/decagon_pytorch/splits.py @@ -1,7 +1,40 @@ import torch +from .data import Data, \ + AdjListData -def train_val_test_split_adj_mat(adj_mat, train_ratio, val_ratio, test_ratio): +def train_val_test_split_adj_mat(adj_mat, train_ratio, val_ratio, test_ratio, + return_edges=False): + + if train_ratio + val_ratio + test_ratio != 1.0: + raise ValueError('Train, validation and test ratios must add up to 1') + + edges = torch.nonzero(adj_mat) + order = torch.randperm(len(edges)) + edges = edges[order, :] + n = round(len(edges) * train_ratio) + edges_train = edges[:n] + n_1 = round(len(edges) * (train_ratio + val_ratio)) + edges_val = edges[n:n_1] + edges_test = edges[n_1:] + + adj_mat_train = torch.zeros_like(adj_mat) + adj_mat_train[edges_train[:, 0], edges_train[:, 1]] = 1 + + adj_mat_val = torch.zeros_like(adj_mat) + adj_mat_val[edges_val[:, 0], edges_val[:, 1]] = 1 + + adj_mat_test = torch.zeros_like(adj_mat) + adj_mat_test[edges_test[:, 0], edges_test[:, 1]] = 1 + + res = (adj_mat_train, adj_mat_val, adj_mat_test) + if return_edges: + res += (edges_train, edges_val, edges_test) + + return res + + +def train_val_test_split_edges(adj_mat, train_ratio, val_ratio, test_ratio): if train_ratio + val_ratio + test_ratio != 1.0: raise ValueError('Train, validation and test ratios must add up to 1') @@ -23,4 +56,5 @@ def train_val_test_split_adj_mat(adj_mat, train_ratio, val_ratio, test_ratio): adj_mat_test = torch.zeros_like(adj_mat) adj_mat_test[edges_test[:, 0], edges_test[:, 1]] = 1 - return adj_mat_train, adj_mat_val, adj_mat_test + return adj_mat_train, adj_mat_val, adj_mat_test, \ + edges_train, edges_val, edges_test -- 2.26.2 From a272a17f2379a7b19cb6e3ec3399ae9c6ab992ce Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 5 Jun 2020 17:52:38 +0200 Subject: [PATCH 054/227] Work on trainprep. --- src/decagon_pytorch/data/trainprep.py | 79 ++++++++++++++++++++++++++- src/decagon_pytorch/normalize.py | 21 +++++++ 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/src/decagon_pytorch/data/trainprep.py b/src/decagon_pytorch/data/trainprep.py index 18373ea..b2b0a49 100644 --- a/src/decagon_pytorch/data/trainprep.py +++ b/src/decagon_pytorch/data/trainprep.py @@ -1,2 +1,77 @@ -def trainprep(data): - pass +from .sampling import fixed_unigram_candidate_sampler +import torch + + +def train_val_test_split_edges(edges, ratios): + train_ratio, val_ratio, test_ratio = ratios + + if train_ratio + val_ratio + test_ratio != 1.0: + raise ValueError('Train, validation and test ratios must add up to 1') + + order = torch.randperm(len(edges)) + edges = edges[order, :] + n = round(len(edges) * train_ratio) + edges_train = edges[:n] + n_1 = round(len(edges) * (train_ratio + val_ratio)) + edges_val = edges[n:n_1] + edges_test = edges[n_1:] + + return edges_train, edges_val, edges_test + + +def prepare_adj_mat(adj_mat, ratios): + degrees = adj_mat.sum(0) + edges_pos = torch.nonzero(adj_mat) + + neg_neighbors = fixed_unigram_candidate_sampler(edges_pos[:, 1], + len(edges), degrees, 0.75) + edges_neg = torch.cat((edges_pos[:, 0], neg_neighbors.view(-1, 1)), 1) + + edges_pos = (edges_pos_train, edges_pos_val, edges_pos_test) = \ + train_val_test_split_edges(edges_pos, ratios) + edges_neg = (edges_neg_train, edges_neg_val, edges_neg_test) = \ + train_val_test_split_edges(edges_neg, ratios) + + return edges_pos, edges_neg + + +class PreparedRelation(object): + def __init__(self, node_type_row, node_type_column, + adj_mat_train, adj_mat_train_trans, + edges_pos, edges_neg, edges_pos_trans, edges_neg_trans): + + self.adj_mat_train = adj_mat_train + self.adj_mat_train_trans = adj_mat_train_trans + self.edges_pos = edges_pos + self.edges_neg = edges_neg + self.edges_pos_trans = edges_pos_trans + self.edges_neg_trans = edges_neg_trans + + +def prepare_relation(r, ratios): + adj_mat = r.get_adjacency_matrix(r.node_type_row, r.node_type_column) + edges_pos, edges_neg = prepare_adj_mat(adj_mat) + + # adj_mat_train = torch.zeros_like(adj_mat) + # adj_mat_train[edges_pos[0][:, 0], edges_pos[0][:, 0]] = 1 + adj_mat_train = torch.sparse_coo_tensor(indices = edges_pos[0].transpose(0, 1), + values=torch.ones(len(edges_pos[0]), dtype=adj_mat.dtype)) + + if r.node_type_row != r.node_type_col: + adj_mat_trans = r.get_adjacency_matrix(r.node_type_col, r.node_type_row) + edges_pos_trans, edges_neg_trans = prepare_adj_mat(adj_mat_trans) + adj_mat_train_trans = torch.sparse_coo_tensor(indices = edges_pos_trans[0].transpose(0, 1), + values=torch.ones(len(edges_pos_trans[0]), dtype=adj_mat_trans.dtype)) + else: + adj_mat_train_trans = adj_mat_trans = \ + edge_pos_trans = edge_neg_trans = None + + return PreparedRelation(r.node_type_row, r.node_type_column, + adj_mat_train, adj_mat_trans_train, + edges_pos, edges_neg, edges_pos_trans, edges_neg_trans) + + +def prepare_training(data): + for (node_type_row, node_type_column), rels in data.relation_types: + for r in rels: + prep_relation_edges() diff --git a/src/decagon_pytorch/normalize.py b/src/decagon_pytorch/normalize.py index 7eb7150..a1af15c 100644 --- a/src/decagon_pytorch/normalize.py +++ b/src/decagon_pytorch/normalize.py @@ -33,3 +33,24 @@ def normalize_adjacency_matrix(adj): coldegree_mat_inv = sp.diags(np.nan_to_num(np.power(colsum, -0.5)).flatten()) adj_normalized = rowdegree_mat_inv.dot(adj).dot(coldegree_mat_inv).tocoo() return sparse_to_tuple(adj_normalized) + + +def norm_adj_mat_one_node_type(adj): + adj = sp.coo_matrix(adj) + assert adj.shape[0] == adj.shape[1]: + adj_ = adj + sp.eye(adj.shape[0]) + rowsum = np.array(adj_.sum(1)) + degree_mat_inv_sqrt = np.power(rowsum, -0.5).flatten() + degree_mat_inv_sqrt = sp.diags(degree_mat_inv_sqrt) + adj_normalized = adj_.dot(degree_mat_inv_sqrt).transpose().dot(degree_mat_inv_sqrt) + return adj_normalized + + +def norm_adj_mat_two_node_types(adj): + adj = sp.coo_matrix(adj) + rowsum = np.array(adj.sum(1)) + colsum = np.array(adj.sum(0)) + rowdegree_mat_inv = sp.diags(np.nan_to_num(np.power(rowsum, -0.5)).flatten()) + coldegree_mat_inv = sp.diags(np.nan_to_num(np.power(colsum, -0.5)).flatten()) + adj_normalized = rowdegree_mat_inv.dot(adj).dot(coldegree_mat_inv).tocoo() + return adj_normalized -- 2.26.2 From 456a037e58775c80f9af32be12a997965a12476d Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 5 Jun 2020 18:07:05 +0200 Subject: [PATCH 055/227] Fix typo. --- src/decagon_pytorch/normalize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decagon_pytorch/normalize.py b/src/decagon_pytorch/normalize.py index a1af15c..4af86cd 100644 --- a/src/decagon_pytorch/normalize.py +++ b/src/decagon_pytorch/normalize.py @@ -37,7 +37,7 @@ def normalize_adjacency_matrix(adj): def norm_adj_mat_one_node_type(adj): adj = sp.coo_matrix(adj) - assert adj.shape[0] == adj.shape[1]: + assert adj.shape[0] == adj.shape[1] adj_ = adj + sp.eye(adj.shape[0]) rowsum = np.array(adj_.sum(1)) degree_mat_inv_sqrt = np.power(rowsum, -0.5).flatten() -- 2.26.2 From 3795545674daa2538657600ee32693e029f91ad8 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 6 Jun 2020 11:52:44 +0200 Subject: [PATCH 056/227] Start icosagon. --- requirements.txt | 3 + src/icosagon/__init__.py | 1 + src/icosagon/data.py | 127 ++++++++++++++++++++++++++++++++++++ tests/icosagon/test_data.py | 49 ++++++++++++++ 4 files changed, 180 insertions(+) create mode 100644 requirements.txt create mode 100644 src/icosagon/__init__.py create mode 100644 src/icosagon/data.py create mode 100644 tests/icosagon/test_data.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..89425dd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +numpy +torch +dataclasses diff --git a/src/icosagon/__init__.py b/src/icosagon/__init__.py new file mode 100644 index 0000000..dc4d081 --- /dev/null +++ b/src/icosagon/__init__.py @@ -0,0 +1 @@ +from .data import Data diff --git a/src/icosagon/data.py b/src/icosagon/data.py new file mode 100644 index 0000000..9f696ca --- /dev/null +++ b/src/icosagon/data.py @@ -0,0 +1,127 @@ +from collections import defaultdict +from dataclasses import dataclass +import torch + + +@dataclass +class NodeType(object): + name: str + count: int + + +@dataclass +class RelationType(object): + name: str + node_type_row: int + node_type_column: int + adjacency_matrix: torch.Tensor + is_autogenerated: bool + + +class Data(object): + def __init__(self) -> None: + self.node_types = [] + self.relation_types = defaultdict(lambda: defaultdict(list)) + + def add_node_type(self, name: str, count: int) -> None: + name = str(name) + count = int(count) + if not name: + raise ValueError('You must provide a non-empty node type name') + if count <= 0: + raise ValueError('You must provide a positive node count') + self.node_types.append(NodeType(name, count)) + + def add_relation_type(self, name: str, node_type_row: int, node_type_column: int, + adjacency_matrix: torch.Tensor, adjacency_matrix_backward: torch.Tensor = None, + two_way: bool = True) -> None: + + name = str(name) + node_type_row = int(node_type_row) + node_type_column = int(node_type_column) + + if node_type_row < 0 or node_type_row > len(self.node_types): + raise ValueError('node_type_row outside of the valid range of node types') + + if node_type_column < 0 or node_type_column > len(self.node_types): + raise ValueError('node_type_column outside of the valid range of node types') + + if not isinstance(adjacency_matrix, torch.Tensor): + raise ValueError('adjacency_matrix must be a torch.Tensor') + + if adjacency_matrix_backward and not isinstance(adjacency_matrix_backward, torch.Tensor): + raise ValueError('adjacency_matrix_backward must be a torch.Tensor') + + if adjacency_matrix.shape != (self.node_types[node_type_row].count, + self.node_types[node_type_column].count): + raise ValueError('adjacency_matrix shape must be (num_row_nodes, num_column_nodes)') + + if adjacency_matrix_backward is not None and \ + adjacency_matrix_backward.shape != (self.node_types[node_type_column].count, + self.node_types[node_type_row].count): + raise ValueError('adjacency_matrix shape must be (num_column_nodes, num_row_nodes)') + + two_way = bool(two_way) + + if node_type_row == node_type_column and \ + adjacency_matrix_backward is not None: + raise ValueError('Relation between nodes of the same type must be expressed using a single matrix') + + self.relation_types[node_type_row][node_type_column].append( + RelationType(name, node_type_row, node_type_column, + adjacency_matrix, False)) + + if node_type_row != node_type_column and two_way: + if adjacency_matrix_backward is None: + adjacency_matrix_backward = adjacency_matrix.transpose(0, 1) + self.relation_types[node_type_column][node_type_row].append( + RelationType(name, node_type_column, node_type_row, + adjacency_matrix_backward, True)) + + def __repr__(self): + n = len(self.node_types) + if n == 0: + return 'Empty Icosagon Data' + s = '' + s += 'Icosagon Data with:\n' + s += '- ' + str(n) + ' node type(s):\n' + for nt in self.node_types: + s += ' - ' + nt.name + '\n' + if len(self.relation_types) == 0: + s += '- No relation types\n' + return s.strip() + + s_1 = '' + count = 0 + for i in range(n): + for j in range(n): + if i not in self.relation_types or \ + j not in self.relation_types[i]: + continue + + s_1 += ' - ' + self.node_types[i].name + ' -- ' + \ + self.node_types[j].name + ':\n' + + for r in self.relation_types[i][j]: + if r.is_autogenerated: + continue + s_1 += ' - ' + r.name + '\n' + count += 1 + + s += '- %d relation type(s):\n' % count + s += s_1 + + return s.strip() + + # n = sum(map(len, self.relation_types)) + # + # for i in range(n): + # for j in range(n): + # key = (i, j) + # if key not in self.relation_types: + # continue + # rels = self.relation_types[key] + # + # for r in rels: + # + # return s.strip() diff --git a/tests/icosagon/test_data.py b/tests/icosagon/test_data.py new file mode 100644 index 0000000..ef64e7b --- /dev/null +++ b/tests/icosagon/test_data.py @@ -0,0 +1,49 @@ +from icosagon import Data +import torch +import pytest + + +def test_data_01(): + d = Data() + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + dummy_0 = torch.zeros((100, 1000)) + dummy_1 = torch.zeros((1000, 100)) + dummy_2 = torch.zeros((100, 100)) + dummy_3 = torch.zeros((1000, 1000)) + d.add_relation_type('Target', 1, 0, dummy_0) + d.add_relation_type('Interaction', 0, 0, dummy_3) + d.add_relation_type('Side Effect: Nausea', 1, 1, dummy_2) + d.add_relation_type('Side Effect: Infertility', 1, 1, dummy_2) + d.add_relation_type('Side Effect: Death', 1, 1, dummy_2) + print(d) + + +def test_data_02(): + d = Data() + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + dummy_0 = torch.zeros((100, 1000)) + dummy_1 = torch.zeros((1000, 100)) + dummy_2 = torch.zeros((100, 100)) + dummy_3 = torch.zeros((1000, 1000)) + with pytest.raises(ValueError): + d.add_relation_type('Target', 1, 0, dummy_1) + with pytest.raises(ValueError): + d.add_relation_type('Interaction', 0, 0, dummy_2) + with pytest.raises(ValueError): + d.add_relation_type('Side Effect: Nausea', 1, 1, dummy_3) + with pytest.raises(ValueError): + d.add_relation_type('Side Effect: Infertility', 1, 1, dummy_3) + with pytest.raises(ValueError): + d.add_relation_type('Side Effect: Death', 1, 1, dummy_3) + print(d) + + +def test_data_03(): + d = Data() + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + with pytest.raises(ValueError): + d.add_relation_type('Target', 1, 0, None) + print(d) -- 2.26.2 From 7ed4bc373a8b8eb36405fc5d4ffe90ff2efd565b Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 6 Jun 2020 13:54:31 +0200 Subject: [PATCH 057/227] Work on icosagon.trainprep. --- docs/decagon-diagram.svg | 258 ++++++++++++++++++++++++++++++- src/icosagon/__init__.py | 6 + src/icosagon/data.py | 6 + src/icosagon/normalize.py | 29 ++++ src/icosagon/sampling.py | 42 +++++ src/icosagon/trainprep.py | 106 +++++++++++++ tests/icosagon/test_trainprep.py | 58 +++++++ 7 files changed, 499 insertions(+), 6 deletions(-) create mode 100644 src/icosagon/normalize.py create mode 100644 src/icosagon/sampling.py create mode 100644 src/icosagon/trainprep.py create mode 100644 tests/icosagon/test_trainprep.py diff --git a/docs/decagon-diagram.svg b/docs/decagon-diagram.svg index 8f8b73a..aafe9ba 100644 --- a/docs/decagon-diagram.svg +++ b/docs/decagon-diagram.svg @@ -18,6 +18,94 @@ sodipodi:docname="decagon-diagram.svg"> + + + + + + + + + + + + + + + + + + @@ -1831,5 +1919,163 @@ x="-6.6224065" id="tspan2683" sodipodi:role="line">A' + + 1 + + 2 + + 3 + + + + [ + [ + 0 0 1 11 0 0 00 1 0 10 1 1 0 A + + 4 + + + + diff --git a/src/icosagon/__init__.py b/src/icosagon/__init__.py index dc4d081..78237bd 100644 --- a/src/icosagon/__init__.py +++ b/src/icosagon/__init__.py @@ -1 +1,7 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + from .data import Data diff --git a/src/icosagon/data.py b/src/icosagon/data.py index 9f696ca..dab3852 100644 --- a/src/icosagon/data.py +++ b/src/icosagon/data.py @@ -1,3 +1,9 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + from collections import defaultdict from dataclasses import dataclass import torch diff --git a/src/icosagon/normalize.py b/src/icosagon/normalize.py new file mode 100644 index 0000000..82e1072 --- /dev/null +++ b/src/icosagon/normalize.py @@ -0,0 +1,29 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import numpy as np +import scipy.sparse as sp + + +def norm_adj_mat_one_node_type(adj): + adj = sp.coo_matrix(adj) + assert adj.shape[0] == adj.shape[1] + adj_ = adj + sp.eye(adj.shape[0]) + rowsum = np.array(adj_.sum(1)) + degree_mat_inv_sqrt = np.power(rowsum, -0.5).flatten() + degree_mat_inv_sqrt = sp.diags(degree_mat_inv_sqrt) + adj_normalized = adj_.dot(degree_mat_inv_sqrt).transpose().dot(degree_mat_inv_sqrt) + return adj_normalized + + +def norm_adj_mat_two_node_types(adj): + adj = sp.coo_matrix(adj) + rowsum = np.array(adj.sum(1)) + colsum = np.array(adj.sum(0)) + rowdegree_mat_inv = sp.diags(np.nan_to_num(np.power(rowsum, -0.5)).flatten()) + coldegree_mat_inv = sp.diags(np.nan_to_num(np.power(colsum, -0.5)).flatten()) + adj_normalized = rowdegree_mat_inv.dot(adj).dot(coldegree_mat_inv).tocoo() + return adj_normalized diff --git a/src/icosagon/sampling.py b/src/icosagon/sampling.py new file mode 100644 index 0000000..8de44f9 --- /dev/null +++ b/src/icosagon/sampling.py @@ -0,0 +1,42 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import numpy as np +import torch +import torch.utils.data +from typing import List, \ + Union + + +def fixed_unigram_candidate_sampler( + true_classes: Union[np.array, torch.Tensor], + num_samples: int, + unigrams: List[Union[int, float]], + distortion: float = 1.): + + if isinstance(true_classes, torch.Tensor): + true_classes = true_classes.detach().cpu().numpy() + if true_classes.shape[0] != num_samples: + raise ValueError('true_classes must be a 2D matrix with shape (num_samples, num_true)') + unigrams = np.array(unigrams) + if distortion != 1.: + unigrams = unigrams.astype(np.float64) ** distortion + # print('unigrams:', unigrams) + indices = np.arange(num_samples) + result = np.zeros(num_samples, dtype=np.int64) + while len(indices) > 0: + # print('len(indices):', len(indices)) + sampler = torch.utils.data.WeightedRandomSampler(unigrams, len(indices)) + candidates = np.array(list(sampler)) + candidates = np.reshape(candidates, (len(indices), 1)) + # print('candidates:', candidates) + # print('true_classes:', true_classes[indices, :]) + result[indices] = candidates.T + mask = (candidates == true_classes[indices, :]) + mask = mask.sum(1).astype(np.bool) + # print('mask:', mask) + indices = indices[mask] + return result diff --git a/src/icosagon/trainprep.py b/src/icosagon/trainprep.py new file mode 100644 index 0000000..38beec6 --- /dev/null +++ b/src/icosagon/trainprep.py @@ -0,0 +1,106 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +from .sampling import fixed_unigram_candidate_sampler +import torch +from dataclasses import dataclass +from typing import Any, \ + List, \ + Tuple, \ + Dict +from .data import NodeType +from collections import defaultdict + + +@dataclass +class TrainValTest(object): + train: Any + val: Any + test: Any + + +@dataclass +class PreparedEdges(object): + positive: TrainValTest + negative: TrainValTest + + +@dataclass +class PreparedRelationType(object): + name: str + node_type_row: int + node_type_column: int + adj_mat_train: torch.Tensor + edges_pos: TrainValTest + edges_neg: TrainValTest + + +@dataclass +class PreparedData(object): + node_types: List[NodeType] + relation_types: Dict[int, Dict[int, List[PreparedRelationType]]] + + +def train_val_test_split_edges(edges: torch.Tensor, + ratios: TrainValTest) -> TrainValTest: + + if not isinstance(edges, torch.Tensor): + raise ValueError('edges must be a torch.Tensor') + + if len(edges.shape) != 2 or edges.shape[1] != 2: + raise ValueError('edges shape must be (num_edges, 2)') + + if not isinstance(ratios, TrainValTest): + raise ValueError('ratios must be a TrainValTest') + + if ratios.train + ratios.val + ratios.test != 1.0: + raise ValueError('Train, validation and test ratios must add up to 1') + + order = torch.randperm(len(edges)) + edges = edges[order, :] + n = round(len(edges) * ratios.train) + edges_train = edges[:n] + n_1 = round(len(edges) * (ratios.train + ratios.val)) + edges_val = edges[n:n_1] + edges_test = edges[n_1:] + + return TrainValTest(edges_train, edges_val, edges_test) + + +def prepare_adj_mat(adj_mat: torch.Tensor, + ratios: TrainValTest) -> Tuple[TrainValTest, TrainValTest]: + + degrees = adj_mat.sum(0) + edges_pos = torch.nonzero(adj_mat) + + neg_neighbors = fixed_unigram_candidate_sampler(edges_pos[:, 1], + len(edges), degrees, 0.75) + edges_neg = torch.cat((edges_pos[:, 0], neg_neighbors.view(-1, 1)), 1) + + edges_pos = train_val_test_split_edges(edges_pos, ratios) + edges_neg = train_val_test_split_edges(edges_neg, ratios) + + return edges_pos, edges_neg + + +def prepare_relation(r, ratios): + adj_mat = r.adjacency_matrix + edges_pos, edges_neg = prepare_adj_mat(adj_mat) + + adj_mat_train = torch.sparse_coo_tensor(indices = edges_pos[0].transpose(0, 1), + values=torch.ones(len(edges_pos[0]), dtype=adj_mat.dtype)) + + return PreparedRelation(r.name, r.node_type_row, r.node_type_column, + adj_mat_train, edges_pos, edges_neg) + + +def prepare_training(data): + relation_types = defaultdict(lambda: defaultdict(list)) + for (node_type_row, node_type_column), rels in data.relation_types: + for r in rels: + relation_types[node_type_row][node_type_column].append( + prep_relation(r)) + return PreparedData(data.node_types, relation_types) diff --git a/tests/icosagon/test_trainprep.py b/tests/icosagon/test_trainprep.py new file mode 100644 index 0000000..874b2c4 --- /dev/null +++ b/tests/icosagon/test_trainprep.py @@ -0,0 +1,58 @@ +from icosagon.trainprep import TrainValTest, \ + train_val_test_split_edges +import torch +import pytest +import numpy as np + + +def test_train_val_test_split_edges_01(): + edges = torch.randint(0, 10, (10, 2)) + with pytest.raises(ValueError): + _ = train_val_test_split_edges(edges, TrainValTest(.5, .5, .5)) + with pytest.raises(ValueError): + _ = train_val_test_split_edges(edges, TrainValTest(.2, .2, .2)) + with pytest.raises(ValueError): + _ = train_val_test_split_edges(edges, None) + with pytest.raises(ValueError): + _ = train_val_test_split_edges(edges, (.8, .1, .1)) + with pytest.raises(ValueError): + _ = train_val_test_split_edges(np.random.randint(0, 10, (10, 2)), TrainValTest(.8, .1, .1)) + with pytest.raises(ValueError): + _ = train_val_test_split_edges(torch.randint(0, 10, (10, 3)), TrainValTest(.8, .1, .1)) + with pytest.raises(ValueError): + _ = train_val_test_split_edges(torch.randint(0, 10, (10, 2, 1)), TrainValTest(.8, .1, .1)) + with pytest.raises(ValueError): + _ = train_val_test_split_edges(None, TrainValTest(.8, .2, .2)) + res = train_val_test_split_edges(edges, TrainValTest(.8, .1, .1)) + assert res.train.shape == (8, 2) and res.val.shape == (1, 2) and \ + res.test.shape == (1, 2) + res = train_val_test_split_edges(edges, TrainValTest(.8, .0, .2)) + assert res.train.shape == (8, 2) and res.val.shape == (0, 2) and \ + res.test.shape == (2, 2) + res = train_val_test_split_edges(edges, TrainValTest(.8, .2, .0)) + assert res.train.shape == (8, 2) and res.val.shape == (2, 2) and \ + res.test.shape == (0, 2) + res = train_val_test_split_edges(edges, TrainValTest(.0, .5, .5)) + assert res.train.shape == (0, 2) and res.val.shape == (5, 2) and \ + res.test.shape == (5, 2) + res = train_val_test_split_edges(edges, TrainValTest(.0, .0, 1.)) + assert res.train.shape == (0, 2) and res.val.shape == (0, 2) and \ + res.test.shape == (10, 2) + res = train_val_test_split_edges(edges, TrainValTest(.0, 1., .0)) + assert res.train.shape == (0, 2) and res.val.shape == (10, 2) and \ + res.test.shape == (0, 2) + + + + # if ratios.train + ratios.val + ratios.test != 1.0: + # raise ValueError('Train, validation and test ratios must add up to 1') + # + # order = torch.randperm(len(edges)) + # edges = edges[order, :] + # n = round(len(edges) * ratios.train) + # edges_train = edges[:n] + # n_1 = round(len(edges) * (ratios.train + ratios.val)) + # edges_val = edges[n:n_1] + # edges_test = edges[n_1:] + # + # return TrainValTest(edges_train, edges_val, edges_test) -- 2.26.2 From 584bf19c3fcb0e9b32515a6c5d19244c199ef76b Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 6 Jun 2020 15:09:24 +0200 Subject: [PATCH 058/227] Work on icosagon.trainprep. --- src/icosagon/sampling.py | 6 +-- src/icosagon/trainprep.py | 43 ++++++++++++---- tests/icosagon/test_data.py | 6 +++ tests/icosagon/test_trainprep.py | 84 +++++++++++++++++++++++++++----- 4 files changed, 114 insertions(+), 25 deletions(-) diff --git a/src/icosagon/sampling.py b/src/icosagon/sampling.py index 8de44f9..28b143d 100644 --- a/src/icosagon/sampling.py +++ b/src/icosagon/sampling.py @@ -13,14 +13,14 @@ from typing import List, \ def fixed_unigram_candidate_sampler( true_classes: Union[np.array, torch.Tensor], - num_samples: int, unigrams: List[Union[int, float]], distortion: float = 1.): if isinstance(true_classes, torch.Tensor): true_classes = true_classes.detach().cpu().numpy() - if true_classes.shape[0] != num_samples: + if len(true_classes.shape) != 2: raise ValueError('true_classes must be a 2D matrix with shape (num_samples, num_true)') + num_samples = true_classes.shape[0] unigrams = np.array(unigrams) if distortion != 1.: unigrams = unigrams.astype(np.float64) ** distortion @@ -39,4 +39,4 @@ def fixed_unigram_candidate_sampler( mask = mask.sum(1).astype(np.bool) # print('mask:', mask) indices = indices[mask] - return result + return torch.tensor(result) diff --git a/src/icosagon/trainprep.py b/src/icosagon/trainprep.py index 38beec6..a999dec 100644 --- a/src/icosagon/trainprep.py +++ b/src/icosagon/trainprep.py @@ -13,6 +13,9 @@ from typing import Any, \ Dict from .data import NodeType from collections import defaultdict +from .normalize import norm_adj_mat_one_node_type, \ + norm_adj_mat_two_node_types +import numpy as np @dataclass @@ -70,28 +73,50 @@ def train_val_test_split_edges(edges: torch.Tensor, return TrainValTest(edges_train, edges_val, edges_test) +def get_edges_and_degrees(adj_mat): + if adj_mat.is_sparse: + adj_mat = adj_mat.coalesce() + degrees = torch.zeros(adj_mat.shape[1], dtype=torch.int64) + degrees = degrees.index_add(0, adj_mat.indices()[1], + torch.ones(adj_mat.indices().shape[1], dtype=torch.int64)) + edges_pos = adj_mat.indices().transpose(0, 1) + else: + degrees = adj_mat.sum(0) + edges_pos = torch.nonzero(adj_mat) + return edges_pos, degrees + + def prepare_adj_mat(adj_mat: torch.Tensor, ratios: TrainValTest) -> Tuple[TrainValTest, TrainValTest]: - degrees = adj_mat.sum(0) - edges_pos = torch.nonzero(adj_mat) + if not isinstance(adj_mat, torch.Tensor): + raise ValueError('adj_mat must be a torch.Tensor') - neg_neighbors = fixed_unigram_candidate_sampler(edges_pos[:, 1], - len(edges), degrees, 0.75) - edges_neg = torch.cat((edges_pos[:, 0], neg_neighbors.view(-1, 1)), 1) + edges_pos, degrees = get_edges_and_degrees(adj_mat) + + neg_neighbors = fixed_unigram_candidate_sampler( + edges_pos[:, 1].view(-1, 1), degrees, 0.75) + print(edges_pos.dtype) + print(neg_neighbors.dtype) + edges_neg = torch.cat((edges_pos[:, 0].view(-1, 1), neg_neighbors.view(-1, 1)), 1) edges_pos = train_val_test_split_edges(edges_pos, ratios) edges_neg = train_val_test_split_edges(edges_neg, ratios) - return edges_pos, edges_neg + adj_mat_train = torch.sparse_coo_tensor(indices = edges_pos.train.transpose(0, 1), + values=torch.ones(len(edges_pos.train), dtype=adj_mat.dtype)) + + return adj_mat_train, edges_pos, edges_neg def prepare_relation(r, ratios): adj_mat = r.adjacency_matrix - edges_pos, edges_neg = prepare_adj_mat(adj_mat) + adj_mat_train, edges_pos, edges_neg = prepare_adj_mat(adj_mat) - adj_mat_train = torch.sparse_coo_tensor(indices = edges_pos[0].transpose(0, 1), - values=torch.ones(len(edges_pos[0]), dtype=adj_mat.dtype)) + if r.node_type_row == r.node_type_column: + adj_mat_train = norm_adj_mat_one_node_type(adj_mat_train) + else: + adj_mat_train = norm_adj_mat_two_node_types(adj_mat_train) return PreparedRelation(r.name, r.node_type_row, r.node_type_column, adj_mat_train, edges_pos, edges_neg) diff --git a/tests/icosagon/test_data.py b/tests/icosagon/test_data.py index ef64e7b..4ec8164 100644 --- a/tests/icosagon/test_data.py +++ b/tests/icosagon/test_data.py @@ -1,3 +1,9 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + from icosagon import Data import torch import pytest diff --git a/tests/icosagon/test_trainprep.py b/tests/icosagon/test_trainprep.py index 874b2c4..ae9c562 100644 --- a/tests/icosagon/test_trainprep.py +++ b/tests/icosagon/test_trainprep.py @@ -1,8 +1,17 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + from icosagon.trainprep import TrainValTest, \ - train_val_test_split_edges + train_val_test_split_edges, \ + get_edges_and_degrees, \ + prepare_adj_mat import torch import pytest import numpy as np +from itertools import chain def test_train_val_test_split_edges_01(): @@ -43,16 +52,65 @@ def test_train_val_test_split_edges_01(): res.test.shape == (0, 2) +def test_train_val_test_split_edges_02(): + edges = torch.randint(0, 30, (30, 2)) + ratios = TrainValTest(.8, .1, .1) + res = train_val_test_split_edges(edges, ratios) + edges = [ tuple(a) for a in edges ] + res = [ tuple(a) for a in chain(res.train, res.val, res.test) ] + assert all([ a in edges for a in res ]) + + +def test_get_edges_and_degrees_01(): + adj_mat_dense = (torch.rand((10, 10)) > .5) + adj_mat_sparse = adj_mat_dense.to_sparse() + edges_dense, degrees_dense = get_edges_and_degrees(adj_mat_dense) + edges_sparse, degrees_sparse = get_edges_and_degrees(adj_mat_sparse) + assert torch.all(degrees_dense == degrees_sparse) + edges_dense = [ tuple(a) for a in edges_dense ] + edges_sparse = [ tuple(a) for a in edges_dense ] + assert len(edges_dense) == len(edges_sparse) + assert all([ a in edges_dense for a in edges_sparse ]) + assert all([ a in edges_sparse for a in edges_dense ]) + # assert torch.all(edges_dense == edges_sparse) + + +def test_prepare_adj_mat_01(): + adj_mat = (torch.rand((10, 10)) > .5) + adj_mat = adj_mat.to_sparse() + ratios = TrainValTest(.8, .1, .1) + _ = prepare_adj_mat(adj_mat, ratios) + + +def test_prepare_adj_mat_02(): + adj_mat = (torch.rand((10, 10)) > .5) + adj_mat = adj_mat.to_sparse() + ratios = TrainValTest(.8, .1, .1) + (adj_mat_train, edges_pos, edges_neg) = prepare_adj_mat(adj_mat, ratios) + assert isinstance(adj_mat_train, torch.Tensor) + assert adj_mat_train.is_sparse + assert adj_mat_train.shape == adj_mat.shape + assert adj_mat_train.dtype == adj_mat.dtype + assert isinstance(edges_pos, TrainValTest) + assert isinstance(edges_neg, TrainValTest) + for a in ['train', 'val', 'test']: + for b in [edges_pos, edges_neg]: + edges = getattr(b, a) + assert isinstance(edges, torch.Tensor) + assert len(edges.shape) == 2 + assert edges.shape[1] == 2 - # if ratios.train + ratios.val + ratios.test != 1.0: - # raise ValueError('Train, validation and test ratios must add up to 1') - # - # order = torch.randperm(len(edges)) - # edges = edges[order, :] - # n = round(len(edges) * ratios.train) - # edges_train = edges[:n] - # n_1 = round(len(edges) * (ratios.train + ratios.val)) - # edges_val = edges[n:n_1] - # edges_test = edges[n_1:] - # - # return TrainValTest(edges_train, edges_val, edges_test) +# def prepare_adj_mat(adj_mat: torch.Tensor, +# ratios: TrainValTest) -> Tuple[TrainValTest, TrainValTest]: +# +# degrees = adj_mat.sum(0) +# edges_pos = torch.nonzero(adj_mat) +# +# neg_neighbors = fixed_unigram_candidate_sampler(edges_pos[:, 1], +# len(edges), degrees, 0.75) +# edges_neg = torch.cat((edges_pos[:, 0], neg_neighbors.view(-1, 1)), 1) +# +# edges_pos = train_val_test_split_edges(edges_pos, ratios) +# edges_neg = train_val_test_split_edges(edges_neg, ratios) +# +# return edges_pos, edges_neg -- 2.26.2 From 05b1ecf47a2b3186906b634492588fcb0df733f1 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 6 Jun 2020 17:19:46 +0200 Subject: [PATCH 059/227] Start rework normalize. --- src/icosagon/data.py | 2 +- src/icosagon/normalize.py | 91 +++++++++++++++++++++++++++--- src/icosagon/trainprep.py | 28 +++++++--- tests/icosagon/test_normalize.py | 95 ++++++++++++++++++++++++++++++++ tests/icosagon/test_trainprep.py | 34 +++++++----- 5 files changed, 220 insertions(+), 30 deletions(-) create mode 100644 tests/icosagon/test_normalize.py diff --git a/src/icosagon/data.py b/src/icosagon/data.py index dab3852..4166d40 100644 --- a/src/icosagon/data.py +++ b/src/icosagon/data.py @@ -21,7 +21,7 @@ class RelationType(object): node_type_row: int node_type_column: int adjacency_matrix: torch.Tensor - is_autogenerated: bool + is_autogenerated: bool = False class Data(object): diff --git a/src/icosagon/normalize.py b/src/icosagon/normalize.py index 82e1072..4d91864 100644 --- a/src/icosagon/normalize.py +++ b/src/icosagon/normalize.py @@ -6,17 +6,90 @@ import numpy as np import scipy.sparse as sp +import torch -def norm_adj_mat_one_node_type(adj): - adj = sp.coo_matrix(adj) - assert adj.shape[0] == adj.shape[1] - adj_ = adj + sp.eye(adj.shape[0]) - rowsum = np.array(adj_.sum(1)) - degree_mat_inv_sqrt = np.power(rowsum, -0.5).flatten() - degree_mat_inv_sqrt = sp.diags(degree_mat_inv_sqrt) - adj_normalized = adj_.dot(degree_mat_inv_sqrt).transpose().dot(degree_mat_inv_sqrt) - return adj_normalized +def add_eye_sparse(adj_mat: torch.Tensor) -> torch.Tensor: + if not isinstance(adj_mat, torch.Tensor): + raise ValueError('adj_mat must be a torch.Tensor') + + if not adj_mat.is_sparse: + raise ValueError('adj_mat must be sparse') + + if len(adj_mat.shape) != 2 or \ + adj_mat.shape[0] != adj_mat.shape[1]: + raise ValueError('adj_mat must be a square matrix') + + adj_mat = adj_mat.coalesce() + indices = adj_mat.indices() + values = adj_mat.values() + + eye_indices = torch.arange(adj_mat.shape[0], dtype=indices.dtype).view(1, -1) + eye_indices = torch.cat((eye_indices, eye_indices), 0) + eye_values = torch.ones(adj_mat.shape[0], dtype=values.dtype) + + indices = torch.cat((indices, eye_indices), 1) + values = torch.cat((values, eye_values), 0) + + adj_mat = torch.sparse_coo_tensor(indices=indices, values=values, size=adj_mat.shape) + + return adj_mat + + +def norm_adj_mat_one_node_type_sparse(adj_mat): + if len(adj_mat.shape) != 2 or \ + adj_mat.shape[0] != adj_mat.shape[1]: + raise ValueError('adj_mat must be a square matrix') + + adj_mat = add_eye_sparse(adj_mat) + + adj_mat = adj_mat.coalesce() + indices = adj_mat.indices() + values = adj_mat.values() + degrees = torch.zeros(adj_mat.shape[0]) + degrees = degrees.index_add(0, indices[0], values.to(degrees.dtype)) + print('degrees:', degrees) + print('values:', values) + values = values.to(degrees.dtype) / degrees[indices[0]] + adj_mat = torch.sparse_coo_tensor(indices=indices, values=values, size=adj_mat.shape) + + return adj_mat + + +def norm_adj_mat_one_node_type_dense(adj_mat): + if not isinstance(adj_mat, torch.Tensor): + raise ValueError('adj_mat must be a torch.Tensor') + + if adj_mat.is_sparse: + raise ValueError('adj_mat must be dense') + + if len(adj_mat.shape) != 2 or \ + adj_mat.shape[0] != adj_mat.shape[1]: + raise ValueError('adj_mat must be a square matrix') + + adj_mat = adj_mat + torch.eye(adj_mat.shape[0], dtype=adj_mat.dtype) + degrees = adj_mat.sum(1).view(-1, 1).to(torch.float32) + adj_mat = adj_mat.to(degrees.dtype) / degrees + + return adj_mat + + +def norm_adj_mat_one_node_type(adj_mat): + if adj_mat.is_sparse: + return norm_adj_mat_one_node_type_sparse(adj_mat) + else: + return norm_adj_mat_one_node_type_dense(adj_mat) + + +# def norm_adj_mat_one_node_type(adj): +# adj = sp.coo_matrix(adj) +# assert adj.shape[0] == adj.shape[1] +# adj_ = adj + sp.eye(adj.shape[0]) +# rowsum = np.array(adj_.sum(1)) +# degree_mat_inv_sqrt = np.power(rowsum, -0.5).flatten() +# degree_mat_inv_sqrt = sp.diags(degree_mat_inv_sqrt) +# adj_normalized = adj_.dot(degree_mat_inv_sqrt).transpose().dot(degree_mat_inv_sqrt) +# return adj_normalized def norm_adj_mat_two_node_types(adj): diff --git a/src/icosagon/trainprep.py b/src/icosagon/trainprep.py index a999dec..e1766f3 100644 --- a/src/icosagon/trainprep.py +++ b/src/icosagon/trainprep.py @@ -11,7 +11,9 @@ from typing import Any, \ List, \ Tuple, \ Dict -from .data import NodeType +from .data import NodeType, \ + RelationType, \ + Data from collections import defaultdict from .normalize import norm_adj_mat_one_node_type, \ norm_adj_mat_two_node_types @@ -73,7 +75,7 @@ def train_val_test_split_edges(edges: torch.Tensor, return TrainValTest(edges_train, edges_val, edges_test) -def get_edges_and_degrees(adj_mat): +def get_edges_and_degrees(adj_mat: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: if adj_mat.is_sparse: adj_mat = adj_mat.coalesce() degrees = torch.zeros(adj_mat.shape[1], dtype=torch.int64) @@ -109,23 +111,35 @@ def prepare_adj_mat(adj_mat: torch.Tensor, return adj_mat_train, edges_pos, edges_neg -def prepare_relation(r, ratios): +def prepare_relation_type(r: RelationType, + ratios: TrainValTest) -> PreparedRelationType: + + if not isinstance(r, RelationType): + raise ValueError('r must be a RelationType') + + if not isinstance(ratios, TrainValTest): + raise ValueError('ratios must be a TrainValTest') + adj_mat = r.adjacency_matrix - adj_mat_train, edges_pos, edges_neg = prepare_adj_mat(adj_mat) + adj_mat_train, edges_pos, edges_neg = prepare_adj_mat(adj_mat, ratios) + print('adj_mat_train:', adj_mat_train) if r.node_type_row == r.node_type_column: adj_mat_train = norm_adj_mat_one_node_type(adj_mat_train) else: adj_mat_train = norm_adj_mat_two_node_types(adj_mat_train) - return PreparedRelation(r.name, r.node_type_row, r.node_type_column, + return PreparedRelationType(r.name, r.node_type_row, r.node_type_column, adj_mat_train, edges_pos, edges_neg) -def prepare_training(data): +def prepare_training(data: Data) -> PreparedData: + if not isinstance(data, Data): + raise ValueError('data must be of class Data') + relation_types = defaultdict(lambda: defaultdict(list)) for (node_type_row, node_type_column), rels in data.relation_types: for r in rels: relation_types[node_type_row][node_type_column].append( - prep_relation(r)) + prep_relation_type(r)) return PreparedData(data.node_types, relation_types) diff --git a/tests/icosagon/test_normalize.py b/tests/icosagon/test_normalize.py new file mode 100644 index 0000000..63452b9 --- /dev/null +++ b/tests/icosagon/test_normalize.py @@ -0,0 +1,95 @@ +from icosagon.normalize import add_eye_sparse, \ + norm_adj_mat_one_node_type_sparse, \ + norm_adj_mat_one_node_type_dense, \ + norm_adj_mat_one_node_type +import decagon_pytorch.normalize +import torch +import pytest +import numpy as np + + +def test_add_eye_sparse_01(): + adj_mat_dense = torch.rand((10, 10)) + adj_mat_sparse = adj_mat_dense.to_sparse() + + adj_mat_dense += torch.eye(10) + adj_mat_sparse = add_eye_sparse(adj_mat_sparse) + + assert torch.all(adj_mat_sparse.to_dense() == adj_mat_dense) + + +def test_add_eye_sparse_02(): + adj_mat_dense = torch.rand((10, 20)) + adj_mat_sparse = adj_mat_dense.to_sparse() + + with pytest.raises(ValueError): + _ = add_eye_sparse(adj_mat_sparse) + + +def test_add_eye_sparse_03(): + adj_mat_dense = torch.rand((10, 10)) + + with pytest.raises(ValueError): + _ = add_eye_sparse(adj_mat_dense) + + +def test_add_eye_sparse_04(): + adj_mat_dense = np.random.rand(10, 10) + + with pytest.raises(ValueError): + _ = add_eye_sparse(adj_mat_dense) + + +def test_norm_adj_mat_one_node_type_sparse_01(): + adj_mat = torch.rand((10, 10)) + adj_mat = (adj_mat > .5) + adj_mat = adj_mat.to_sparse() + _ = norm_adj_mat_one_node_type_sparse(adj_mat) + + +def test_norm_adj_mat_one_node_type_sparse_02(): + adj_mat_dense = torch.rand((10, 10)) + adj_mat_dense = (adj_mat_dense > .5) + adj_mat_sparse = adj_mat_dense.to_sparse() + adj_mat_sparse = norm_adj_mat_one_node_type_sparse(adj_mat_sparse) + adj_mat_dense = norm_adj_mat_one_node_type_dense(adj_mat_dense) + assert torch.all(adj_mat_sparse.to_dense() == adj_mat_dense) + + +def test_norm_adj_mat_one_node_type_dense_01(): + adj_mat = torch.rand((10, 10)) + adj_mat = (adj_mat > .5) + _ = norm_adj_mat_one_node_type_dense(adj_mat) + + +def test_norm_adj_mat_one_node_type_dense_02(): + adj_mat = torch.tensor([ + [0, 1, 1, 0], # 3 + [1, 0, 1, 0], # 3 + [1, 1, 0, 1], # 4 + [0, 0, 1, 0] # 2 + ]) + expect = np.array([ + [1/3, 1/3, 1/3, 0], + [1/3, 1/3, 1/3, 0], + [1/4, 1/4, 1/4, 1/4], + [0, 0, 1/2, 1/2] + ], dtype=np.float32) + res = decagon_pytorch.normalize.norm_adj_mat_one_node_type(adj_mat) + res = res.todense().astype(np.float32) + print('res:', res) + print('expect:', expect) + assert torch.all(res == expect) + + +@pytest.mark.skip +def test_norm_adj_mat_one_node_type_dense_03(): + adj_mat = torch.rand((10, 10)) + adj_mat = (adj_mat > .5) + adj_mat_dec = decagon_pytorch.normalize.norm_adj_mat_one_node_type(adj_mat) + adj_mat_ico = norm_adj_mat_one_node_type_dense(adj_mat) + adj_mat_dec = adj_mat_dec.todense() + adj_mat_ico = adj_mat_ico.detach().cpu().numpy() + print('adj_mat_dec:', adj_mat_dec) + print('adj_mat_ico:', adj_mat_ico) + assert np.all(adj_mat_dec == adj_mat_ico) diff --git a/tests/icosagon/test_trainprep.py b/tests/icosagon/test_trainprep.py index ae9c562..967bb1e 100644 --- a/tests/icosagon/test_trainprep.py +++ b/tests/icosagon/test_trainprep.py @@ -7,11 +7,13 @@ from icosagon.trainprep import TrainValTest, \ train_val_test_split_edges, \ get_edges_and_degrees, \ - prepare_adj_mat + prepare_adj_mat, \ + prepare_relation_type import torch import pytest import numpy as np from itertools import chain +from icosagon.data import RelationType def test_train_val_test_split_edges_01(): @@ -100,17 +102,23 @@ def test_prepare_adj_mat_02(): assert len(edges.shape) == 2 assert edges.shape[1] == 2 -# def prepare_adj_mat(adj_mat: torch.Tensor, -# ratios: TrainValTest) -> Tuple[TrainValTest, TrainValTest]: -# -# degrees = adj_mat.sum(0) -# edges_pos = torch.nonzero(adj_mat) -# -# neg_neighbors = fixed_unigram_candidate_sampler(edges_pos[:, 1], -# len(edges), degrees, 0.75) -# edges_neg = torch.cat((edges_pos[:, 0], neg_neighbors.view(-1, 1)), 1) + +def test_prepare_relation_type_01(): + adj_mat = (torch.rand((10, 10)) > .5) + r = RelationType('Test', 0, 0, adj_mat) + ratios = TrainValTest(.8, .1, .1) + _ = prepare_relation_type(r, ratios) + + + +# def prepare_relation(r, ratios): +# adj_mat = r.adjacency_matrix +# adj_mat_train, edges_pos, edges_neg = prepare_adj_mat(adj_mat) # -# edges_pos = train_val_test_split_edges(edges_pos, ratios) -# edges_neg = train_val_test_split_edges(edges_neg, ratios) +# if r.node_type_row == r.node_type_column: +# adj_mat_train = norm_adj_mat_one_node_type(adj_mat_train) +# else: +# adj_mat_train = norm_adj_mat_two_node_types(adj_mat_train) # -# return edges_pos, edges_neg +# return PreparedRelation(r.name, r.node_type_row, r.node_type_column, +# adj_mat_train, edges_pos, edges_neg) -- 2.26.2 From 6f80b65a46ebf0125cd652b95ea7423be8e52644 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 6 Jun 2020 17:39:35 +0200 Subject: [PATCH 060/227] New implementation for one node type normalization seems to work. --- src/icosagon/normalize.py | 22 +++++++++++++------ tests/icosagon/test_normalize.py | 37 ++++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/src/icosagon/normalize.py b/src/icosagon/normalize.py index 4d91864..3bb9260 100644 --- a/src/icosagon/normalize.py +++ b/src/icosagon/normalize.py @@ -46,11 +46,15 @@ def norm_adj_mat_one_node_type_sparse(adj_mat): adj_mat = adj_mat.coalesce() indices = adj_mat.indices() values = adj_mat.values() - degrees = torch.zeros(adj_mat.shape[0]) - degrees = degrees.index_add(0, indices[0], values.to(degrees.dtype)) - print('degrees:', degrees) - print('values:', values) - values = values.to(degrees.dtype) / degrees[indices[0]] + degrees_row = torch.zeros(adj_mat.shape[0]) + degrees_row = degrees_row.index_add(0, indices[0], values.to(degrees_row.dtype)) + degrees_col = torch.zeros(adj_mat.shape[1]) + degrees_col = degrees_col.index_add(0, indices[1], values.to(degrees_col.dtype)) + # degrees_row = torch.sqrt(degrees_row) + # degrees_col = torch.sqrt(degrees_col) + # print('degrees:', degrees) + # print('values:', values) + values = values.to(degrees_row.dtype) / torch.sqrt(degrees_row[indices[0]] * degrees_col[indices[1]]) adj_mat = torch.sparse_coo_tensor(indices=indices, values=values, size=adj_mat.shape) return adj_mat @@ -68,8 +72,12 @@ def norm_adj_mat_one_node_type_dense(adj_mat): raise ValueError('adj_mat must be a square matrix') adj_mat = adj_mat + torch.eye(adj_mat.shape[0], dtype=adj_mat.dtype) - degrees = adj_mat.sum(1).view(-1, 1).to(torch.float32) - adj_mat = adj_mat.to(degrees.dtype) / degrees + degrees_row = adj_mat.sum(1).view(-1, 1).to(torch.float32) + degrees_col = adj_mat.sum(0).view(1, -1).to(torch.float32) + degrees_row = torch.sqrt(degrees_row) + degrees_col = torch.sqrt(degrees_col) + adj_mat = adj_mat.to(degrees_row.dtype) / degrees_row + adj_mat = adj_mat / degrees_col return adj_mat diff --git a/tests/icosagon/test_normalize.py b/tests/icosagon/test_normalize.py index 63452b9..98cc301 100644 --- a/tests/icosagon/test_normalize.py +++ b/tests/icosagon/test_normalize.py @@ -6,6 +6,7 @@ import decagon_pytorch.normalize import torch import pytest import numpy as np +from math import sqrt def test_add_eye_sparse_01(): @@ -53,7 +54,7 @@ def test_norm_adj_mat_one_node_type_sparse_02(): adj_mat_sparse = adj_mat_dense.to_sparse() adj_mat_sparse = norm_adj_mat_one_node_type_sparse(adj_mat_sparse) adj_mat_dense = norm_adj_mat_one_node_type_dense(adj_mat_dense) - assert torch.all(adj_mat_sparse.to_dense() == adj_mat_dense) + assert torch.all(adj_mat_sparse.to_dense() - adj_mat_dense < 0.000001) def test_norm_adj_mat_one_node_type_dense_01(): @@ -68,28 +69,42 @@ def test_norm_adj_mat_one_node_type_dense_02(): [1, 0, 1, 0], # 3 [1, 1, 0, 1], # 4 [0, 0, 1, 0] # 2 + # 3 3 4 2 ]) - expect = np.array([ - [1/3, 1/3, 1/3, 0], - [1/3, 1/3, 1/3, 0], - [1/4, 1/4, 1/4, 1/4], - [0, 0, 1/2, 1/2] + expect_denom = np.array([ + [ 3, 3, sqrt(3)*2, sqrt(6) ], + [ 3, 3, sqrt(3)*2, sqrt(6) ], + [ sqrt(3)*2, sqrt(3)*2, 4, sqrt(2)*2 ], + [ sqrt(6), sqrt(6), sqrt(2)*2, 2 ] ], dtype=np.float32) + expect = (adj_mat.detach().cpu().numpy().astype(np.float32) + np.eye(4)) / expect_denom + # expect = np.array([ + # [1/3, 1/3, 1/3, 0], + # [1/3, 1/3, 1/3, 0], + # [1/4, 1/4, 1/4, 1/4], + # [0, 0, 1/2, 1/2] + # ], dtype=np.float32) res = decagon_pytorch.normalize.norm_adj_mat_one_node_type(adj_mat) res = res.todense().astype(np.float32) print('res:', res) print('expect:', expect) - assert torch.all(res == expect) + assert np.all(res - expect < 0.000001) -@pytest.mark.skip def test_norm_adj_mat_one_node_type_dense_03(): - adj_mat = torch.rand((10, 10)) - adj_mat = (adj_mat > .5) + # adj_mat = torch.rand((10, 10)) + adj_mat = torch.tensor([ + [0, 1, 1, 0, 0], + [1, 0, 1, 0, 1], + [1, 1, 0, .5, .5], + [0, 0, .5, 0, 1], + [0, 1, .5, 1, 0] + ]) + # adj_mat = (adj_mat > .5) adj_mat_dec = decagon_pytorch.normalize.norm_adj_mat_one_node_type(adj_mat) adj_mat_ico = norm_adj_mat_one_node_type_dense(adj_mat) adj_mat_dec = adj_mat_dec.todense() adj_mat_ico = adj_mat_ico.detach().cpu().numpy() print('adj_mat_dec:', adj_mat_dec) print('adj_mat_ico:', adj_mat_ico) - assert np.all(adj_mat_dec == adj_mat_ico) + assert np.all(adj_mat_dec - adj_mat_ico < 0.000001) -- 2.26.2 From 706ff0b009e2f7b4376d42c19cb65da5f9bc6ea6 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 6 Jun 2020 19:05:19 +0200 Subject: [PATCH 061/227] New implementation of normalize for one/two node types and sparse/dense - good. --- src/icosagon/normalize.py | 109 ++++++++++++++++++------------- tests/icosagon/test_normalize.py | 77 +++++++++++++++++++++- 2 files changed, 141 insertions(+), 45 deletions(-) diff --git a/src/icosagon/normalize.py b/src/icosagon/normalize.py index 3bb9260..ae63c51 100644 --- a/src/icosagon/normalize.py +++ b/src/icosagon/normalize.py @@ -9,17 +9,37 @@ import scipy.sparse as sp import torch -def add_eye_sparse(adj_mat: torch.Tensor) -> torch.Tensor: +def _check_tensor(adj_mat): if not isinstance(adj_mat, torch.Tensor): raise ValueError('adj_mat must be a torch.Tensor') + +def _check_sparse(adj_mat): if not adj_mat.is_sparse: raise ValueError('adj_mat must be sparse') + +def _check_dense(adj_mat): + if adj_mat.is_sparse: + raise ValueError('adj_mat must be dense') + + +def _check_square(adj_mat): if len(adj_mat.shape) != 2 or \ adj_mat.shape[0] != adj_mat.shape[1]: raise ValueError('adj_mat must be a square matrix') + +def _check_2d(adj_mat): + if len(adj_mat.shape) != 2: + raise ValueError('adj_mat must be a square matrix') + + +def add_eye_sparse(adj_mat: torch.Tensor) -> torch.Tensor: + _check_tensor(adj_mat) + _check_sparse(adj_mat) + _check_square(adj_mat) + adj_mat = adj_mat.coalesce() indices = adj_mat.indices() values = adj_mat.values() @@ -36,12 +56,42 @@ def add_eye_sparse(adj_mat: torch.Tensor) -> torch.Tensor: return adj_mat -def norm_adj_mat_one_node_type_sparse(adj_mat): - if len(adj_mat.shape) != 2 or \ - adj_mat.shape[0] != adj_mat.shape[1]: - raise ValueError('adj_mat must be a square matrix') +def norm_adj_mat_one_node_type_sparse(adj_mat: torch.Tensor) -> torch.Tensor: + _check_tensor(adj_mat) + _check_sparse(adj_mat) + _check_square(adj_mat) adj_mat = add_eye_sparse(adj_mat) + adj_mat = norm_adj_mat_two_node_types_sparse(adj_mat) + + return adj_mat + + +def norm_adj_mat_one_node_type_dense(adj_mat: torch.Tensor) -> torch.Tensor: + _check_tensor(adj_mat) + _check_dense(adj_mat) + _check_square(adj_mat) + + adj_mat = adj_mat + torch.eye(adj_mat.shape[0], dtype=adj_mat.dtype) + adj_mat = norm_adj_mat_two_node_types_dense(adj_mat) + + return adj_mat + + +def norm_adj_mat_one_node_type(adj_mat: torch.Tensor) -> torch.Tensor: + _check_tensor(adj_mat) + _check_square(adj_mat) + + if adj_mat.is_sparse: + return norm_adj_mat_one_node_type_sparse(adj_mat) + else: + return norm_adj_mat_one_node_type_dense(adj_mat) + + +def norm_adj_mat_two_node_types_sparse(adj_mat: torch.Tensor) -> torch.Tensor: + _check_tensor(adj_mat) + _check_sparse(adj_mat) + _check_2d(adj_mat) adj_mat = adj_mat.coalesce() indices = adj_mat.indices() @@ -50,28 +100,17 @@ def norm_adj_mat_one_node_type_sparse(adj_mat): degrees_row = degrees_row.index_add(0, indices[0], values.to(degrees_row.dtype)) degrees_col = torch.zeros(adj_mat.shape[1]) degrees_col = degrees_col.index_add(0, indices[1], values.to(degrees_col.dtype)) - # degrees_row = torch.sqrt(degrees_row) - # degrees_col = torch.sqrt(degrees_col) - # print('degrees:', degrees) - # print('values:', values) values = values.to(degrees_row.dtype) / torch.sqrt(degrees_row[indices[0]] * degrees_col[indices[1]]) adj_mat = torch.sparse_coo_tensor(indices=indices, values=values, size=adj_mat.shape) return adj_mat -def norm_adj_mat_one_node_type_dense(adj_mat): - if not isinstance(adj_mat, torch.Tensor): - raise ValueError('adj_mat must be a torch.Tensor') - - if adj_mat.is_sparse: - raise ValueError('adj_mat must be dense') - - if len(adj_mat.shape) != 2 or \ - adj_mat.shape[0] != adj_mat.shape[1]: - raise ValueError('adj_mat must be a square matrix') +def norm_adj_mat_two_node_types_dense(adj_mat: torch.Tensor) -> torch.Tensor: + _check_tensor(adj_mat) + _check_dense(adj_mat) + _check_2d(adj_mat) - adj_mat = adj_mat + torch.eye(adj_mat.shape[0], dtype=adj_mat.dtype) degrees_row = adj_mat.sum(1).view(-1, 1).to(torch.float32) degrees_col = adj_mat.sum(0).view(1, -1).to(torch.float32) degrees_row = torch.sqrt(degrees_row) @@ -82,29 +121,11 @@ def norm_adj_mat_one_node_type_dense(adj_mat): return adj_mat -def norm_adj_mat_one_node_type(adj_mat): +def norm_adj_mat_two_node_types(adj_mat: torch.Tensor) -> torch.Tensor: + _check_tensor(adj_mat) + _check_2d(adj_mat) + if adj_mat.is_sparse: - return norm_adj_mat_one_node_type_sparse(adj_mat) + return norm_adj_mat_two_node_types_sparse(adj_mat) else: - return norm_adj_mat_one_node_type_dense(adj_mat) - - -# def norm_adj_mat_one_node_type(adj): -# adj = sp.coo_matrix(adj) -# assert adj.shape[0] == adj.shape[1] -# adj_ = adj + sp.eye(adj.shape[0]) -# rowsum = np.array(adj_.sum(1)) -# degree_mat_inv_sqrt = np.power(rowsum, -0.5).flatten() -# degree_mat_inv_sqrt = sp.diags(degree_mat_inv_sqrt) -# adj_normalized = adj_.dot(degree_mat_inv_sqrt).transpose().dot(degree_mat_inv_sqrt) -# return adj_normalized - - -def norm_adj_mat_two_node_types(adj): - adj = sp.coo_matrix(adj) - rowsum = np.array(adj.sum(1)) - colsum = np.array(adj.sum(0)) - rowdegree_mat_inv = sp.diags(np.nan_to_num(np.power(rowsum, -0.5)).flatten()) - coldegree_mat_inv = sp.diags(np.nan_to_num(np.power(colsum, -0.5)).flatten()) - adj_normalized = rowdegree_mat_inv.dot(adj).dot(coldegree_mat_inv).tocoo() - return adj_normalized + return norm_adj_mat_two_node_types_dense(adj_mat) diff --git a/tests/icosagon/test_normalize.py b/tests/icosagon/test_normalize.py index 98cc301..e4b0fef 100644 --- a/tests/icosagon/test_normalize.py +++ b/tests/icosagon/test_normalize.py @@ -1,7 +1,10 @@ from icosagon.normalize import add_eye_sparse, \ norm_adj_mat_one_node_type_sparse, \ norm_adj_mat_one_node_type_dense, \ - norm_adj_mat_one_node_type + norm_adj_mat_one_node_type, \ + norm_adj_mat_two_node_types_sparse, \ + norm_adj_mat_two_node_types_dense, \ + norm_adj_mat_two_node_types import decagon_pytorch.normalize import torch import pytest @@ -108,3 +111,75 @@ def test_norm_adj_mat_one_node_type_dense_03(): print('adj_mat_dec:', adj_mat_dec) print('adj_mat_ico:', adj_mat_ico) assert np.all(adj_mat_dec - adj_mat_ico < 0.000001) + + +def test_norm_adj_mat_two_node_types_sparse_01(): + adj_mat = torch.rand((10, 20)) + adj_mat = (adj_mat > .5) + adj_mat = adj_mat.to_sparse() + _ = norm_adj_mat_two_node_types_sparse(adj_mat) + + +def test_norm_adj_mat_two_node_types_sparse_02(): + adj_mat_dense = torch.rand((10, 20)) + adj_mat_dense = (adj_mat_dense > .5) + adj_mat_sparse = adj_mat_dense.to_sparse() + adj_mat_sparse = norm_adj_mat_two_node_types_sparse(adj_mat_sparse) + adj_mat_dense = norm_adj_mat_two_node_types_dense(adj_mat_dense) + assert torch.all(adj_mat_sparse.to_dense() - adj_mat_dense < 0.000001) + + +def test_norm_adj_mat_two_node_types_dense_01(): + adj_mat = torch.rand((10, 20)) + adj_mat = (adj_mat > .5) + _ = norm_adj_mat_two_node_types_dense(adj_mat) + + +def test_norm_adj_mat_two_node_types_dense_02(): + adj_mat = torch.tensor([ + [0, 1, 1, 0], # 2 + [1, 0, 1, 0], # 2 + [1, 1, 0, 1], # 3 + [0, 0, 1, 0] # 1 + # 2 2 3 1 + ]) + expect_denom = np.array([ + [ 2, 2, sqrt(6), sqrt(2) ], + [ 2, 2, sqrt(6), sqrt(2) ], + [ sqrt(6), sqrt(6), 3, sqrt(3) ], + [ sqrt(2), sqrt(2), sqrt(3), 1 ] + ], dtype=np.float32) + expect = adj_mat.detach().cpu().numpy().astype(np.float32) / expect_denom + res = decagon_pytorch.normalize.norm_adj_mat_two_node_types(adj_mat) + res = res.todense().astype(np.float32) + print('res:', res) + print('expect:', expect) + assert np.all(res - expect < 0.000001) + + +def test_norm_adj_mat_two_node_types_dense_03(): + adj_mat = torch.tensor([ + [0, 1, 1, 0, 0], + [1, 0, 1, 0, 1], + [1, 1, 0, .5, .5], + [0, 0, .5, 0, 1], + [0, 1, .5, 1, 0] + ]) + adj_mat_dec = decagon_pytorch.normalize.norm_adj_mat_two_node_types(adj_mat) + adj_mat_ico = norm_adj_mat_two_node_types_dense(adj_mat) + adj_mat_dec = adj_mat_dec.todense() + adj_mat_ico = adj_mat_ico.detach().cpu().numpy() + print('adj_mat_dec:', adj_mat_dec) + print('adj_mat_ico:', adj_mat_ico) + assert np.all(adj_mat_dec - adj_mat_ico < 0.000001) + + +def test_norm_adj_mat_two_node_types_dense_04(): + adj_mat = torch.rand((10, 20)) + adj_mat_dec = decagon_pytorch.normalize.norm_adj_mat_two_node_types(adj_mat) + adj_mat_ico = norm_adj_mat_two_node_types_dense(adj_mat) + adj_mat_dec = adj_mat_dec.todense() + adj_mat_ico = adj_mat_ico.detach().cpu().numpy() + print('adj_mat_dec:', adj_mat_dec) + print('adj_mat_ico:', adj_mat_ico) + assert np.all(adj_mat_dec - adj_mat_ico < 0.000001) -- 2.26.2 From d503fb5738ab5f5759850c4d2d61f7a993d3078d Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sun, 7 Jun 2020 10:21:14 +0200 Subject: [PATCH 062/227] Bring more elements to icosagon. --- src/icosagon/convolve.py | 85 +++++++++++++++++++++++++++++ src/icosagon/dropout.py | 33 ++++++++++++ src/icosagon/weights.py | 19 +++++++ tests/icosagon/test_convolve.py | 94 +++++++++++++++++++++++++++++++++ tests/icosagon/test_dropout.py | 26 +++++++++ 5 files changed, 257 insertions(+) create mode 100644 src/icosagon/convolve.py create mode 100644 src/icosagon/dropout.py create mode 100644 src/icosagon/weights.py create mode 100644 tests/icosagon/test_convolve.py create mode 100644 tests/icosagon/test_dropout.py diff --git a/src/icosagon/convolve.py b/src/icosagon/convolve.py new file mode 100644 index 0000000..08dada4 --- /dev/null +++ b/src/icosagon/convolve.py @@ -0,0 +1,85 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import torch +from .dropout import dropout_sparse, \ + dropout_dense +from .weights import init_glorot +from typing import List, Callable + + +class GraphConv(torch.nn.Module): + """Convolution layer for sparse AND dense inputs.""" + def __init__(self, in_channels: int, out_channels: int, + adjacency_matrix: torch.Tensor, **kwargs) -> None: + super().__init__(**kwargs) + self.in_channels = in_channels + self.out_channels = out_channels + self.weight = init_glorot(in_channels, out_channels) + self.adjacency_matrix = adjacency_matrix + + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = torch.sparse.mm(x, self.weight) \ + if x.is_sparse \ + else torch.mm(x, self.weight) + x = torch.sparse.mm(self.adjacency_matrix, x) \ + if self.adjacency_matrix.is_sparse \ + else torch.mm(self.adjacency_matrix, x) + return x + + +class DropoutGraphConvActivation(torch.nn.Module): + def __init__(self, input_dim: int, output_dim: int, + adjacency_matrix: torch.Tensor, keep_prob: float=1., + activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, + **kwargs) -> None: + super().__init__(**kwargs) + self.input_dim = input_dim + self.output_dim = output_dim + self.adjacency_matrix = adjacency_matrix + self.keep_prob = keep_prob + self.activation = activation + self.graph_conv = GraphConv(input_dim, output_dim, adjacency_matrix) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = dropout_sparse(x, self.keep_prob) \ + if x.is_sparse \ + else dropout_dense(x, self.keep_prob) + x = self.graph_conv(x) + x = self.activation(x) + return x + + +class MultiDGCA(torch.nn.Module): + def __init__(self, input_dim: List[int], output_dim: int, + adjacency_matrices: List[torch.Tensor], keep_prob: float=1., + activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, + **kwargs) -> None: + super().__init__(**kwargs) + self.input_dim = input_dim + self.output_dim = output_dim + self.adjacency_matrices = adjacency_matrices + self.keep_prob = keep_prob + self.activation = activation + self.dgca = None + self.build() + + def build(self): + if len(self.input_dim) != len(self.adjacency_matrices): + raise ValueError('input_dim must have the same length as adjacency_matrices') + self.dgca = [] + for input_dim, adj_mat in zip(self.input_dim, self.adjacency_matrices): + self.dgca.append(DenseDropoutGraphConvActivation(input_dim, self.output_dim, adj_mat, self.keep_prob, self.activation)) + + def forward(self, x: List[torch.Tensor]) -> List[torch.Tensor]: + if not isinstance(x, list): + raise ValueError('x must be a list of tensors') + out = torch.zeros(len(x[0]), self.output_dim, dtype=x[0].dtype) + for i, f in enumerate(self.dgca): + out += f(x[i]) + out = torch.nn.functional.normalize(out, p=2, dim=1) + return out diff --git a/src/icosagon/dropout.py b/src/icosagon/dropout.py new file mode 100644 index 0000000..13f086a --- /dev/null +++ b/src/icosagon/dropout.py @@ -0,0 +1,33 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import torch + + +def dropout_sparse(x, keep_prob): + x = x.coalesce() + i = x._indices() + v = x._values() + size = x.size() + + n = keep_prob + torch.rand(len(v)) + n = torch.floor(n).to(torch.bool) + i = i[:,n] + v = v[n] + x = torch.sparse_coo_tensor(i, v, size=size) + + return x * (1./keep_prob) + + +def dropout_dense(x, keep_prob): + x = x.clone().detach() + i = torch.nonzero(x) + + n = keep_prob + torch.rand(len(i)) + n = (1. - torch.floor(n)).to(torch.bool) + x[i[n, 0], i[n, 1]] = 0. + + return x * (1./keep_prob) diff --git a/src/icosagon/weights.py b/src/icosagon/weights.py new file mode 100644 index 0000000..2dcb7b4 --- /dev/null +++ b/src/icosagon/weights.py @@ -0,0 +1,19 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import torch +import numpy as np + + +def init_glorot(in_channels, out_channels, dtype=torch.float32): + """Create a weight variable with Glorot & Bengio (AISTATS 2010) + initialization. + """ + init_range = np.sqrt(6.0 / (in_channels + out_channels)) + initial = -init_range + 2 * init_range * \ + torch.rand(( in_channels, out_channels ), dtype=dtype) + initial = initial.requires_grad_(True) + return initial diff --git a/tests/icosagon/test_convolve.py b/tests/icosagon/test_convolve.py new file mode 100644 index 0000000..d124ab1 --- /dev/null +++ b/tests/icosagon/test_convolve.py @@ -0,0 +1,94 @@ +from icosagon.convolve import GraphConv, \ + DropoutGraphConvActivation, \ + MultiDGCA +import torch + + +def _test_graph_conv_01(use_sparse: bool): + adj_mat = torch.rand((10, 20)) + adj_mat[adj_mat < .5] = 0 + adj_mat = torch.ceil(adj_mat) + + node_reprs = torch.eye(20) + + graph_conv = GraphConv(20, 20, adj_mat.to_sparse() \ + if use_sparse else adj_mat) + graph_conv.weight = torch.eye(20) + + res = graph_conv(node_reprs) + assert torch.all(res == adj_mat) + + +def _test_graph_conv_02(use_sparse: bool): + adj_mat = torch.rand((10, 20)) + adj_mat[adj_mat < .5] = 0 + adj_mat = torch.ceil(adj_mat) + + node_reprs = torch.eye(20) + + graph_conv = GraphConv(20, 20, adj_mat.to_sparse() \ + if use_sparse else adj_mat) + graph_conv.weight = torch.eye(20) * 2 + + res = graph_conv(node_reprs) + assert torch.all(res == adj_mat * 2) + + +def _test_graph_conv_03(use_sparse: bool): + adj_mat = torch.tensor([ + [1, 0, 1, 0, 1, 0], # [1, 0, 0] + [1, 0, 1, 0, 0, 1], # [1, 0, 0] + [1, 1, 0, 1, 0, 0], # [0, 1, 0] + [0, 0, 0, 1, 0, 1], # [0, 1, 0] + [1, 1, 1, 1, 1, 1], # [0, 0, 1] + [0, 0, 0, 1, 1, 1] # [0, 0, 1] + ], dtype=torch.float32) + + expect = torch.tensor([ + [1, 1, 1], + [1, 1, 1], + [2, 1, 0], + [0, 1, 1], + [2, 2, 2], + [0, 1, 2] + ], dtype=torch.float32) + + node_reprs = torch.eye(6) + + graph_conv = GraphConv(6, 3, adj_mat.to_sparse() \ + if use_sparse else adj_mat) + graph_conv.weight = torch.tensor([ + [1, 0, 0], + [1, 0, 0], + [0, 1, 0], + [0, 1, 0], + [0, 0, 1], + [0, 0, 1] + ], dtype=torch.float32) + + res = graph_conv(node_reprs) + assert torch.all(res == expect) + + +def test_graph_conv_dense_01(): + _test_graph_conv_01(use_sparse=False) + + +def test_graph_conv_dense_02(): + _test_graph_conv_02(use_sparse=False) + + +def test_graph_conv_dense_03(): + _test_graph_conv_03(use_sparse=False) + + +def test_graph_conv_sparse_01(): + _test_graph_conv_01(use_sparse=True) + + +def test_graph_conv_sparse_02(): + _test_graph_conv_02(use_sparse=True) + + +def test_graph_conv_sparse_03(): + _test_graph_conv_03(use_sparse=True) diff --git a/tests/icosagon/test_dropout.py b/tests/icosagon/test_dropout.py new file mode 100644 index 0000000..374f0a9 --- /dev/null +++ b/tests/icosagon/test_dropout.py @@ -0,0 +1,26 @@ +from icosagon.dropout import dropout_sparse, \ + dropout_dense +import torch +import numpy as np + + +def test_dropout_01(): + for i in range(11): + torch.random.manual_seed(i) + a = torch.rand((5, 10)) + a[a < .5] = 0 + + keep_prob=i/10. + np.finfo(np.float32).eps + + torch.random.manual_seed(i) + b = dropout_dense(a, keep_prob=keep_prob) + + torch.random.manual_seed(i) + c = dropout_sparse(a.to_sparse(), keep_prob=keep_prob) + + print('keep_prob:', keep_prob) + print('a:', a.detach().cpu().numpy()) + print('b:', b.detach().cpu().numpy()) + print('c:', c, c.to_dense().detach().cpu().numpy()) + + assert torch.all(b == c.to_dense()) -- 2.26.2 From 07a82e0da9665dc993cbd62730395463e6a687ab Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sun, 7 Jun 2020 12:05:29 +0200 Subject: [PATCH 063/227] Work on icosagon. --- src/icosagon/convolve.py | 39 +--------- src/icosagon/decode.py | 131 ++++++++++++++++++++++++++++++++ src/icosagon/dropout.py | 7 ++ src/icosagon/layer.py | 94 +++++++++++++++++++++++ src/icosagon/trainprep.py | 2 +- tests/icosagon/test_convolve.py | 100 +++++++++++++++++++++++- 6 files changed, 333 insertions(+), 40 deletions(-) create mode 100644 src/icosagon/decode.py create mode 100644 src/icosagon/layer.py diff --git a/src/icosagon/convolve.py b/src/icosagon/convolve.py index 08dada4..118a58e 100644 --- a/src/icosagon/convolve.py +++ b/src/icosagon/convolve.py @@ -5,14 +5,12 @@ import torch -from .dropout import dropout_sparse, \ - dropout_dense +from .dropout import dropout from .weights import init_glorot from typing import List, Callable class GraphConv(torch.nn.Module): - """Convolution layer for sparse AND dense inputs.""" def __init__(self, in_channels: int, out_channels: int, adjacency_matrix: torch.Tensor, **kwargs) -> None: super().__init__(**kwargs) @@ -46,40 +44,7 @@ class DropoutGraphConvActivation(torch.nn.Module): self.graph_conv = GraphConv(input_dim, output_dim, adjacency_matrix) def forward(self, x: torch.Tensor) -> torch.Tensor: - x = dropout_sparse(x, self.keep_prob) \ - if x.is_sparse \ - else dropout_dense(x, self.keep_prob) + x = dropout(x, self.keep_prob) x = self.graph_conv(x) x = self.activation(x) return x - - -class MultiDGCA(torch.nn.Module): - def __init__(self, input_dim: List[int], output_dim: int, - adjacency_matrices: List[torch.Tensor], keep_prob: float=1., - activation: Callable[[torch.Tensor], torch.Tensor]=torch.nn.functional.relu, - **kwargs) -> None: - super().__init__(**kwargs) - self.input_dim = input_dim - self.output_dim = output_dim - self.adjacency_matrices = adjacency_matrices - self.keep_prob = keep_prob - self.activation = activation - self.dgca = None - self.build() - - def build(self): - if len(self.input_dim) != len(self.adjacency_matrices): - raise ValueError('input_dim must have the same length as adjacency_matrices') - self.dgca = [] - for input_dim, adj_mat in zip(self.input_dim, self.adjacency_matrices): - self.dgca.append(DenseDropoutGraphConvActivation(input_dim, self.output_dim, adj_mat, self.keep_prob, self.activation)) - - def forward(self, x: List[torch.Tensor]) -> List[torch.Tensor]: - if not isinstance(x, list): - raise ValueError('x must be a list of tensors') - out = torch.zeros(len(x[0]), self.output_dim, dtype=x[0].dtype) - for i, f in enumerate(self.dgca): - out += f(x[i]) - out = torch.nn.functional.normalize(out, p=2, dim=1) - return out diff --git a/src/icosagon/decode.py b/src/icosagon/decode.py new file mode 100644 index 0000000..16efd83 --- /dev/null +++ b/src/icosagon/decode.py @@ -0,0 +1,131 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import torch +from .weights import init_glorot +from .dropout import dropout + + +class DEDICOMDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, drop_prob=0., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.drop_prob = drop_prob + self.activation = activation + + self.global_interaction = init_glorot(input_dim, input_dim) + self.local_variation = [ + torch.flatten(init_glorot(input_dim, 1)) \ + for _ in range(num_relation_types) + ] + + def forward(self, inputs_row, inputs_col): + outputs = [] + for k in range(self.num_relation_types): + inputs_row = dropout(inputs_row, 1.-self.drop_prob) + inputs_col = dropout(inputs_col, 1.-self.drop_prob) + + relation = torch.diag(self.local_variation[k]) + + product1 = torch.mm(inputs_row, relation) + product2 = torch.mm(product1, self.global_interaction) + product3 = torch.mm(product2, relation) + rec = torch.bmm(product3.view(product3.shape[0], 1, product3.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) + outputs.append(self.activation(rec)) + return outputs + + +class DistMultDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, drop_prob=0., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.drop_prob = drop_prob + self.activation = activation + + self.relation = [ + torch.flatten(init_glorot(input_dim, 1)) \ + for _ in range(num_relation_types) + ] + + def forward(self, inputs_row, inputs_col): + outputs = [] + for k in range(self.num_relation_types): + inputs_row = dropout(inputs_row, 1.-self.drop_prob) + inputs_col = dropout(inputs_col, 1.-self.drop_prob) + + relation = torch.diag(self.relation[k]) + + intermediate_product = torch.mm(inputs_row, relation) + rec = torch.bmm(intermediate_product.view(intermediate_product.shape[0], 1, intermediate_product.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) + outputs.append(self.activation(rec)) + return outputs + + +class BilinearDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, drop_prob=0., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.drop_prob = drop_prob + self.activation = activation + + self.relation = [ + init_glorot(input_dim, input_dim) \ + for _ in range(num_relation_types) + ] + + def forward(self, inputs_row, inputs_col): + outputs = [] + for k in range(self.num_relation_types): + inputs_row = dropout(inputs_row, 1.-self.drop_prob) + inputs_col = dropout(inputs_col, 1.-self.drop_prob) + + intermediate_product = torch.mm(inputs_row, self.relation[k]) + rec = torch.bmm(intermediate_product.view(intermediate_product.shape[0], 1, intermediate_product.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) + outputs.append(self.activation(rec)) + return outputs + + +class InnerProductDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, drop_prob=0., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.drop_prob = drop_prob + self.activation = activation + + + def forward(self, inputs_row, inputs_col): + outputs = [] + for k in range(self.num_relation_types): + inputs_row = dropout(inputs_row, 1.-self.drop_prob) + inputs_col = dropout(inputs_col, 1.-self.drop_prob) + + rec = torch.bmm(inputs_row.view(inputs_row.shape[0], 1, inputs_row.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) + outputs.append(self.activation(rec)) + return outputs diff --git a/src/icosagon/dropout.py b/src/icosagon/dropout.py index 13f086a..95d0575 100644 --- a/src/icosagon/dropout.py +++ b/src/icosagon/dropout.py @@ -31,3 +31,10 @@ def dropout_dense(x, keep_prob): x[i[n, 0], i[n, 1]] = 0. return x * (1./keep_prob) + + +def dropout(x, keep_prob): + if x.is_sparse: + return dropout_sparse(x, keep_prob) + else: + return dropout_dense(x, keep_prob) diff --git a/src/icosagon/layer.py b/src/icosagon/layer.py new file mode 100644 index 0000000..aef27ea --- /dev/null +++ b/src/icosagon/layer.py @@ -0,0 +1,94 @@ +import torch +from .convolve import DropoutGraphConvActivation +from .data import Data +from .trainprep import PreparedData +from typing import List, \ + Union, \ + Callable +from collections import defaultdict +from dataclasses import dataclass + + +@dataclass +class Convolutions(object): + node_type_column: int + convolutions: List[DropoutGraphConvActivation] + + +class DecagonLayer(torch.nn.Module): + def __init__(self, + input_dim: List[int], + output_dim: List[int], + data: Union[Data, PreparedData], + keep_prob: float = 1., + rel_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, + layer_activation: Callable[[torch.Tensor], torch.Tensor] = torch.nn.functional.relu, + **kwargs): + + super().__init__(**kwargs) + + if not isinstance(input_dim, list): + raise ValueError('input_dim must be a list') + + if not isinstance(output_dim, list): + raise ValueError('output_dim must be a list') + + if not isinstance(data, Data) and not isinstance(data, PreparedData): + raise ValueError('data must be of type Data or PreparedData') + + self.input_dim = input_dim + self.output_dim = output_dim + self.data = data + self.keep_prob = float(keep_prob) + self.rel_activation = rel_activation + self.layer_activation = layer_activation + + self.is_sparse = False + self.next_layer_repr = None + self.build() + + def build(self): + n = len(self.data.node_types) + rel_types = self.data.relation_types + + self.next_layer_repr = [ [] for _ in range(n) ] + + for node_type_row in range(n): + if node_type_row not in rel_types: + continue + + for node_type_column in range(n): + if node_type_column not in rel_types[node_type_row]: + continue + + rels = rel_types[node_type_row][node_type_column] + if len(rels) == 0: + continue + + convolutions = [] + + for r in rels: + conv = DropoutGraphConvActivation(self.input_dim[node_type_column], + self.output_dim[node_type_row], r.adjacency_matrix, + self.keep_prob, self.rel_activation) + + convolutions.append(conv) + + self.next_layer_repr[node_type_row].append( + Convolutions(node_type_column, convolutions)) + + def __call__(self, prev_layer_repr): + next_layer_repr = [ [] for _ in range(len(self.data.node_types)) ] + n = len(self.data.node_types) + + for node_type_row in range(n): + for convolutions in self.next_layer_repr[node_type_row]: + repr_ = [ conv(prev_layer_repr[convolutions.node_type_column]) \ + for conv in convolutions.convolutions ] + repr_ = sum(repr_) + repr_ = torch.nn.functional.normalize(repr_, p=2, dim=1) + next_layer_repr[i].append(repr_) + next_layer_repr[i] = sum(next_layer_repr[i]) + next_layer_repr[i] = self.layer_activation(next_layer_repr[i]) + + return next_layer_repr diff --git a/src/icosagon/trainprep.py b/src/icosagon/trainprep.py index e1766f3..10886a1 100644 --- a/src/icosagon/trainprep.py +++ b/src/icosagon/trainprep.py @@ -38,7 +38,7 @@ class PreparedRelationType(object): name: str node_type_row: int node_type_column: int - adj_mat_train: torch.Tensor + adjacency_matrix: torch.Tensor edges_pos: TrainValTest edges_neg: TrainValTest diff --git a/tests/icosagon/test_convolve.py b/tests/icosagon/test_convolve.py index d124ab1..a916a89 100644 --- a/tests/icosagon/test_convolve.py +++ b/tests/icosagon/test_convolve.py @@ -1,7 +1,7 @@ from icosagon.convolve import GraphConv, \ - DropoutGraphConvActivation, \ - MultiDGCA + DropoutGraphConvActivation import torch +from icosagon.dropout import dropout def _test_graph_conv_01(use_sparse: bool): @@ -92,3 +92,99 @@ def test_graph_conv_sparse_02(): def test_graph_conv_sparse_03(): _test_graph_conv_03(use_sparse=True) + + +def _test_dropout_graph_conv_activation_01(use_sparse: bool): + adj_mat = torch.rand((10, 20)) + adj_mat[adj_mat < .5] = 0 + adj_mat = torch.ceil(adj_mat) + node_reprs = torch.eye(20) + + conv_1 = DropoutGraphConvActivation(20, 20, adj_mat.to_sparse() \ + if use_sparse else adj_mat, keep_prob=1., + activation=lambda x: x) + + conv_2 = GraphConv(20, 20, adj_mat.to_sparse() \ + if use_sparse else adj_mat) + conv_2.weight = conv_1.graph_conv.weight + + res_1 = conv_1(node_reprs) + res_2 = conv_2(node_reprs) + + print('res_1:', res_1.detach().cpu().numpy()) + print('res_2:', res_2.detach().cpu().numpy()) + + assert torch.all(res_1 == res_2) + + +def _test_dropout_graph_conv_activation_02(use_sparse: bool): + adj_mat = torch.rand((10, 20)) + adj_mat[adj_mat < .5] = 0 + adj_mat = torch.ceil(adj_mat) + node_reprs = torch.eye(20) + + conv_1 = DropoutGraphConvActivation(20, 20, adj_mat.to_sparse() \ + if use_sparse else adj_mat, keep_prob=1., + activation=lambda x: x * 2) + + conv_2 = GraphConv(20, 20, adj_mat.to_sparse() \ + if use_sparse else adj_mat) + conv_2.weight = conv_1.graph_conv.weight + + res_1 = conv_1(node_reprs) + res_2 = conv_2(node_reprs) + + print('res_1:', res_1.detach().cpu().numpy()) + print('res_2:', res_2.detach().cpu().numpy()) + + assert torch.all(res_1 == res_2 * 2) + + +def _test_dropout_graph_conv_activation_03(use_sparse: bool): + adj_mat = torch.rand((10, 20)) + adj_mat[adj_mat < .5] = 0 + adj_mat = torch.ceil(adj_mat) + node_reprs = torch.eye(20) + + conv_1 = DropoutGraphConvActivation(20, 20, adj_mat.to_sparse() \ + if use_sparse else adj_mat, keep_prob=.5, + activation=lambda x: x) + + conv_2 = GraphConv(20, 20, adj_mat.to_sparse() \ + if use_sparse else adj_mat) + conv_2.weight = conv_1.graph_conv.weight + + torch.random.manual_seed(0) + res_1 = conv_1(node_reprs) + + torch.random.manual_seed(0) + res_2 = conv_2(dropout(node_reprs, 0.5)) + + print('res_1:', res_1.detach().cpu().numpy()) + print('res_2:', res_2.detach().cpu().numpy()) + + assert torch.all(res_1 == res_2) + + +def test_dropout_graph_conv_activation_dense_01(): + _test_dropout_graph_conv_activation_01(False) + + +def test_dropout_graph_conv_activation_sparse_01(): + _test_dropout_graph_conv_activation_01(True) + + +def test_dropout_graph_conv_activation_dense_02(): + _test_dropout_graph_conv_activation_02(False) + + +def test_dropout_graph_conv_activation_sparse_02(): + _test_dropout_graph_conv_activation_02(True) + + +def test_dropout_graph_conv_activation_dense_03(): + _test_dropout_graph_conv_activation_03(False) + + +def test_dropout_graph_conv_activation_sparse_03(): + _test_dropout_graph_conv_activation_03(True) -- 2.26.2 From d4dd1f29230a0b05fc27a04d806b487cac147ec8 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sun, 7 Jun 2020 12:51:22 +0200 Subject: [PATCH 064/227] Add input to icosagon. --- src/icosagon/{layer.py => convlayer.py} | 0 src/icosagon/declayer.py | 2 + src/icosagon/input.py | 76 +++++++++++++++++ tests/icosagon/test_input.py | 106 ++++++++++++++++++++++++ 4 files changed, 184 insertions(+) rename src/icosagon/{layer.py => convlayer.py} (100%) create mode 100644 src/icosagon/declayer.py create mode 100644 src/icosagon/input.py create mode 100644 tests/icosagon/test_input.py diff --git a/src/icosagon/layer.py b/src/icosagon/convlayer.py similarity index 100% rename from src/icosagon/layer.py rename to src/icosagon/convlayer.py diff --git a/src/icosagon/declayer.py b/src/icosagon/declayer.py new file mode 100644 index 0000000..46bffca --- /dev/null +++ b/src/icosagon/declayer.py @@ -0,0 +1,2 @@ +# from .layer import DecagonLayer +# from .input import OneHotInputLayer diff --git a/src/icosagon/input.py b/src/icosagon/input.py new file mode 100644 index 0000000..c0b2672 --- /dev/null +++ b/src/icosagon/input.py @@ -0,0 +1,76 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import torch +from typing import Union, \ + List +from .data import Data + + +class InputLayer(torch.nn.Module): + def __init__(self, data: Data, output_dim: Union[int, List[int]] = None, **kwargs) -> None: + output_dim = output_dim or \ + list(map(lambda a: a.count, data.node_types)) + if not isinstance(output_dim, list): + output_dim = [output_dim,] * len(data.node_types) + + super().__init__(**kwargs) + self.output_dim = output_dim + self.data = data + + self.is_sparse=False + self.node_reps = None + self.build() + + def build(self) -> None: + self.node_reps = [] + for i, nt in enumerate(self.data.node_types): + reps = torch.rand(nt.count, self.output_dim[i]) + reps = torch.nn.Parameter(reps) + self.register_parameter('node_reps[%d]' % i, reps) + self.node_reps.append(reps) + + def forward(self, x) -> List[torch.nn.Parameter]: + return self.node_reps + + def __repr__(self) -> str: + s = '' + s += 'Icosagon input layer with output_dim: %s\n' % self.output_dim + s += ' # of node types: %d\n' % len(self.data.node_types) + for nt in self.data.node_types: + s += ' - %s (%d)\n' % (nt.name, nt.count) + return s.strip() + + +class OneHotInputLayer(torch.nn.Module): + def __init__(self, data: Data, **kwargs) -> None: + output_dim = [ a.count for a in data.node_types ] + super().__init__(**kwargs) + self.output_dim = output_dim + self.data = data + + self.is_sparse=True + self.node_reps = None + self.build() + + def build(self) -> None: + self.node_reps = [] + for i, nt in enumerate(self.data.node_types): + reps = torch.eye(nt.count).to_sparse() + reps = torch.nn.Parameter(reps) + self.register_parameter('node_reps[%d]' % i, reps) + self.node_reps.append(reps) + + def forward(self, x) -> List[torch.nn.Parameter]: + return self.node_reps + + def __repr__(self) -> str: + s = '' + s += 'One-hot Icosagon input layer\n' + s += ' # of node types: %d\n' % len(self.data.node_types) + for nt in self.data.node_types: + s += ' - %s (%d)\n' % (nt.name, nt.count) + return s.strip() diff --git a/tests/icosagon/test_input.py b/tests/icosagon/test_input.py new file mode 100644 index 0000000..3e73ce1 --- /dev/null +++ b/tests/icosagon/test_input.py @@ -0,0 +1,106 @@ +from icosagon.input import InputLayer, \ + OneHotInputLayer +from icosagon.data import Data +import torch +import pytest + + +def _some_data(): + d = Data() + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + d.add_relation_type('Target', 1, 0, torch.rand(100, 1000)) + d.add_relation_type('Interaction', 0, 0, torch.rand(1000, 1000)) + d.add_relation_type('Side Effect: Nausea', 1, 1, torch.rand(100, 100)) + d.add_relation_type('Side Effect: Infertility', 1, 1, torch.rand(100, 100)) + d.add_relation_type('Side Effect: Death', 1, 1, torch.rand(100, 100)) + return d + + +def _some_data_with_interactions(): + d = Data() + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + d.add_relation_type('Target', 1, 0, + torch.rand((100, 1000), dtype=torch.float32).round()) + d.add_relation_type('Interaction', 0, 0, + torch.rand((1000, 1000), dtype=torch.float32).round()) + d.add_relation_type('Side Effect: Nausea', 1, 1, + torch.rand((100, 100), dtype=torch.float32).round()) + d.add_relation_type('Side Effect: Infertility', 1, 1, + torch.rand((100, 100), dtype=torch.float32).round()) + d.add_relation_type('Side Effect: Death', 1, 1, + torch.rand((100, 100), dtype=torch.float32).round()) + return d + + +def test_input_layer_01(): + d = _some_data() + for output_dim in [32, 64, 128]: + layer = InputLayer(d, output_dim) + assert layer.output_dim[0] == output_dim + assert len(layer.node_reps) == 2 + assert layer.node_reps[0].shape == (1000, output_dim) + assert layer.node_reps[1].shape == (100, output_dim) + assert layer.data == d + + +def test_input_layer_02(): + d = _some_data() + layer = InputLayer(d, 32) + res = layer(None) + assert isinstance(res[0], torch.Tensor) + assert isinstance(res[1], torch.Tensor) + assert res[0].shape == (1000, 32) + assert res[1].shape == (100, 32) + assert torch.all(res[0] == layer.node_reps[0]) + assert torch.all(res[1] == layer.node_reps[1]) + + +def test_input_layer_03(): + if torch.cuda.device_count() == 0: + pytest.skip('No CUDA devices on this host') + d = _some_data() + layer = InputLayer(d, 32) + device = torch.device('cuda:0') + layer = layer.to(device) + print(list(layer.parameters())) + # assert layer.device.type == 'cuda:0' + assert layer.node_reps[0].device == device + assert layer.node_reps[1].device == device + + +def test_one_hot_input_layer_01(): + d = _some_data() + layer = OneHotInputLayer(d) + assert layer.output_dim == [1000, 100] + assert len(layer.node_reps) == 2 + assert layer.node_reps[0].shape == (1000, 1000) + assert layer.node_reps[1].shape == (100, 100) + assert layer.data == d + assert layer.is_sparse + + +def test_one_hot_input_layer_02(): + d = _some_data() + layer = OneHotInputLayer(d) + res = layer(None) + assert isinstance(res[0], torch.Tensor) + assert isinstance(res[1], torch.Tensor) + assert res[0].shape == (1000, 1000) + assert res[1].shape == (100, 100) + assert torch.all(res[0].to_dense() == layer.node_reps[0].to_dense()) + assert torch.all(res[1].to_dense() == layer.node_reps[1].to_dense()) + + +def test_one_hot_input_layer_03(): + if torch.cuda.device_count() == 0: + pytest.skip('No CUDA devices on this host') + d = _some_data() + layer = OneHotInputLayer(d) + device = torch.device('cuda:0') + layer = layer.to(device) + print(list(layer.parameters())) + # assert layer.device.type == 'cuda:0' + assert layer.node_reps[0].device == device + assert layer.node_reps[1].device == device -- 2.26.2 From a305dc70aa8118fe654c34b836f451dba99e86a2 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sun, 7 Jun 2020 13:40:10 +0200 Subject: [PATCH 065/227] Add tests for convlayer. --- src/decagon_pytorch/convolve/universal.py | 2 +- src/icosagon/convlayer.py | 11 +- src/icosagon/input.py | 5 +- tests/icosagon/test_convlayer.py | 167 ++++++++++++++++++++++ 4 files changed, 179 insertions(+), 6 deletions(-) create mode 100644 tests/icosagon/test_convlayer.py diff --git a/src/decagon_pytorch/convolve/universal.py b/src/decagon_pytorch/convolve/universal.py index f39d1a8..b266448 100644 --- a/src/decagon_pytorch/convolve/universal.py +++ b/src/decagon_pytorch/convolve/universal.py @@ -73,7 +73,7 @@ class MultiDGCA(torch.nn.Module): raise ValueError('input_dim must have the same length as adjacency_matrices') self.dgca = [] for input_dim, adj_mat in zip(self.input_dim, self.adjacency_matrices): - self.dgca.append(DenseDropoutGraphConvActivation(input_dim, self.output_dim, adj_mat, self.keep_prob, self.activation)) + self.dgca.append(DropoutGraphConvActivation(input_dim, self.output_dim, adj_mat, self.keep_prob, self.activation)) def forward(self, x: List[torch.Tensor]) -> List[torch.Tensor]: if not isinstance(x, list): diff --git a/src/icosagon/convlayer.py b/src/icosagon/convlayer.py index aef27ea..88f15b8 100644 --- a/src/icosagon/convlayer.py +++ b/src/icosagon/convlayer.py @@ -30,8 +30,11 @@ class DecagonLayer(torch.nn.Module): if not isinstance(input_dim, list): raise ValueError('input_dim must be a list') + if not output_dim: + raise ValueError('output_dim must be specified') + if not isinstance(output_dim, list): - raise ValueError('output_dim must be a list') + output_dim = [output_dim] * len(data.node_types) if not isinstance(data, Data) and not isinstance(data, PreparedData): raise ValueError('data must be of type Data or PreparedData') @@ -87,8 +90,8 @@ class DecagonLayer(torch.nn.Module): for conv in convolutions.convolutions ] repr_ = sum(repr_) repr_ = torch.nn.functional.normalize(repr_, p=2, dim=1) - next_layer_repr[i].append(repr_) - next_layer_repr[i] = sum(next_layer_repr[i]) - next_layer_repr[i] = self.layer_activation(next_layer_repr[i]) + next_layer_repr[node_type_row].append(repr_) + next_layer_repr[node_type_row] = sum(next_layer_repr[node_type_row]) + next_layer_repr[node_type_row] = self.layer_activation(next_layer_repr[node_type_row]) return next_layer_repr diff --git a/src/icosagon/input.py b/src/icosagon/input.py index c0b2672..4f7dd73 100644 --- a/src/icosagon/input.py +++ b/src/icosagon/input.py @@ -11,9 +11,12 @@ from .data import Data class InputLayer(torch.nn.Module): - def __init__(self, data: Data, output_dim: Union[int, List[int]] = None, **kwargs) -> None: + def __init__(self, data: Data, output_dim: Union[int, List[int]] = None, + **kwargs) -> None: + output_dim = output_dim or \ list(map(lambda a: a.count, data.node_types)) + if not isinstance(output_dim, list): output_dim = [output_dim,] * len(data.node_types) diff --git a/tests/icosagon/test_convlayer.py b/tests/icosagon/test_convlayer.py new file mode 100644 index 0000000..82b7f56 --- /dev/null +++ b/tests/icosagon/test_convlayer.py @@ -0,0 +1,167 @@ +from icosagon.input import InputLayer, \ + OneHotInputLayer +from icosagon.convlayer import DecagonLayer, \ + Convolutions +from icosagon.data import Data +import torch +import pytest +from icosagon.convolve import DropoutGraphConvActivation +from decagon_pytorch.convolve import MultiDGCA +import decagon_pytorch.convolve + + +def _some_data_with_interactions(): + d = Data() + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + d.add_relation_type('Target', 1, 0, + torch.rand((100, 1000), dtype=torch.float32).round()) + d.add_relation_type('Interaction', 0, 0, + torch.rand((1000, 1000), dtype=torch.float32).round()) + d.add_relation_type('Side Effect: Nausea', 1, 1, + torch.rand((100, 100), dtype=torch.float32).round()) + d.add_relation_type('Side Effect: Infertility', 1, 1, + torch.rand((100, 100), dtype=torch.float32).round()) + d.add_relation_type('Side Effect: Death', 1, 1, + torch.rand((100, 100), dtype=torch.float32).round()) + return d + + +def test_decagon_layer_01(): + d = _some_data_with_interactions() + in_layer = InputLayer(d) + d_layer = DecagonLayer(in_layer.output_dim, 32, d) + seq = torch.nn.Sequential(in_layer, d_layer) + _ = seq(None) # dummy call + + +def test_decagon_layer_02(): + d = _some_data_with_interactions() + in_layer = OneHotInputLayer(d) + d_layer = DecagonLayer(in_layer.output_dim, 32, d) + seq = torch.nn.Sequential(in_layer, d_layer) + _ = seq(None) # dummy call + + +def test_decagon_layer_03(): + d = _some_data_with_interactions() + in_layer = OneHotInputLayer(d) + d_layer = DecagonLayer(in_layer.output_dim, 32, d) + + assert d_layer.input_dim == [ 1000, 100 ] + assert d_layer.output_dim == [ 32, 32 ] + assert d_layer.data == d + assert d_layer.keep_prob == 1. + assert d_layer.rel_activation(0.5) == 0.5 + x = torch.tensor([-1, 0, 0.5, 1]) + assert (d_layer.layer_activation(x) == torch.nn.functional.relu(x)).all() + + assert not d_layer.is_sparse + assert len(d_layer.next_layer_repr) == 2 + + for i in range(2): + assert len(d_layer.next_layer_repr[i]) == 2 + assert isinstance(d_layer.next_layer_repr[i], list) + assert isinstance(d_layer.next_layer_repr[i][0], Convolutions) + assert isinstance(d_layer.next_layer_repr[i][0].node_type_column, int) + assert isinstance(d_layer.next_layer_repr[i][0].convolutions, list) + assert all([ + isinstance(dgca, DropoutGraphConvActivation) \ + for dgca in d_layer.next_layer_repr[i][0].convolutions + ]) + assert all([ + dgca.output_dim == 32 \ + for dgca in d_layer.next_layer_repr[i][0].convolutions + ]) + + +def test_decagon_layer_04(): + # check if it is equivalent to MultiDGCA, as it should be + + d = Data() + d.add_node_type('Dummy', 100) + d.add_relation_type('Dummy Relation', 0, 0, + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + + in_layer = OneHotInputLayer(d) + + multi_dgca = MultiDGCA([10], 32, + [r.adjacency_matrix for r in d.relation_types[0][0]], + keep_prob=1., activation=lambda x: x) + + d_layer = DecagonLayer(in_layer.output_dim, 32, d, + keep_prob=1., rel_activation=lambda x: x, + layer_activation=lambda x: x) + + assert isinstance(d_layer.next_layer_repr[0][0].convolutions[0], + DropoutGraphConvActivation) + + weight = d_layer.next_layer_repr[0][0].convolutions[0].graph_conv.weight + assert isinstance(weight, torch.Tensor) + + assert len(multi_dgca.dgca) == 1 + assert isinstance(multi_dgca.dgca[0], + decagon_pytorch.convolve.DropoutGraphConvActivation) + + multi_dgca.dgca[0].graph_conv.weight = weight + + seq = torch.nn.Sequential(in_layer, d_layer) + out_d_layer = seq(None) + out_multi_dgca = multi_dgca(in_layer(None)) + + assert isinstance(out_d_layer, list) + assert len(out_d_layer) == 1 + + assert torch.all(out_d_layer[0] == out_multi_dgca) + + +def test_decagon_layer_05(): + # check if it is equivalent to MultiDGCA, as it should be + # this time for two relations, same edge type + + d = Data() + d.add_node_type('Dummy', 100) + d.add_relation_type('Dummy Relation 1', 0, 0, + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + d.add_relation_type('Dummy Relation 2', 0, 0, + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + + in_layer = OneHotInputLayer(d) + + multi_dgca = MultiDGCA([100, 100], 32, + [r.adjacency_matrix for r in d.relation_types[0][0]], + keep_prob=1., activation=lambda x: x) + + d_layer = DecagonLayer(in_layer.output_dim, output_dim=32, data=d, + keep_prob=1., rel_activation=lambda x: x, + layer_activation=lambda x: x) + + assert all([ + isinstance(dgca, DropoutGraphConvActivation) \ + for dgca in d_layer.next_layer_repr[0][0].convolutions + ]) + + weight = [ dgca.graph_conv.weight \ + for dgca in d_layer.next_layer_repr[0][0].convolutions ] + assert all([ + isinstance(w, torch.Tensor) \ + for w in weight + ]) + + assert len(multi_dgca.dgca) == 2 + for i in range(2): + assert isinstance(multi_dgca.dgca[i], + decagon_pytorch.convolve.DropoutGraphConvActivation) + + for i in range(2): + multi_dgca.dgca[i].graph_conv.weight = weight[i] + + seq = torch.nn.Sequential(in_layer, d_layer) + out_d_layer = seq(None) + x = in_layer(None) + out_multi_dgca = multi_dgca([ x[0], x[0] ]) + + assert isinstance(out_d_layer, list) + assert len(out_d_layer) == 1 + + assert torch.all(out_d_layer[0] == out_multi_dgca) -- 2.26.2 From dd0eb812514cf05828bef7e11e09a715c6f62910 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sun, 7 Jun 2020 14:06:56 +0200 Subject: [PATCH 066/227] Add tests for decode. --- src/icosagon/decode.py | 32 ++++++------- tests/icosagon/test_decode.py | 86 +++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 16 deletions(-) create mode 100644 tests/icosagon/test_decode.py diff --git a/src/icosagon/decode.py b/src/icosagon/decode.py index 16efd83..dccf508 100644 --- a/src/icosagon/decode.py +++ b/src/icosagon/decode.py @@ -11,13 +11,13 @@ from .dropout import dropout class DEDICOMDecoder(torch.nn.Module): """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" - def __init__(self, input_dim, num_relation_types, drop_prob=0., + def __init__(self, input_dim, num_relation_types, keep_prob=1., activation=torch.sigmoid, **kwargs): super().__init__(**kwargs) self.input_dim = input_dim self.num_relation_types = num_relation_types - self.drop_prob = drop_prob + self.keep_prob = keep_prob self.activation = activation self.global_interaction = init_glorot(input_dim, input_dim) @@ -29,8 +29,8 @@ class DEDICOMDecoder(torch.nn.Module): def forward(self, inputs_row, inputs_col): outputs = [] for k in range(self.num_relation_types): - inputs_row = dropout(inputs_row, 1.-self.drop_prob) - inputs_col = dropout(inputs_col, 1.-self.drop_prob) + inputs_row = dropout(inputs_row, self.keep_prob) + inputs_col = dropout(inputs_col, self.keep_prob) relation = torch.diag(self.local_variation[k]) @@ -46,13 +46,13 @@ class DEDICOMDecoder(torch.nn.Module): class DistMultDecoder(torch.nn.Module): """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" - def __init__(self, input_dim, num_relation_types, drop_prob=0., + def __init__(self, input_dim, num_relation_types, keep_prob=1., activation=torch.sigmoid, **kwargs): super().__init__(**kwargs) self.input_dim = input_dim self.num_relation_types = num_relation_types - self.drop_prob = drop_prob + self.keep_prob = keep_prob self.activation = activation self.relation = [ @@ -63,8 +63,8 @@ class DistMultDecoder(torch.nn.Module): def forward(self, inputs_row, inputs_col): outputs = [] for k in range(self.num_relation_types): - inputs_row = dropout(inputs_row, 1.-self.drop_prob) - inputs_col = dropout(inputs_col, 1.-self.drop_prob) + inputs_row = dropout(inputs_row, self.keep_prob) + inputs_col = dropout(inputs_col, self.keep_prob) relation = torch.diag(self.relation[k]) @@ -78,13 +78,13 @@ class DistMultDecoder(torch.nn.Module): class BilinearDecoder(torch.nn.Module): """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" - def __init__(self, input_dim, num_relation_types, drop_prob=0., + def __init__(self, input_dim, num_relation_types, keep_prob=1., activation=torch.sigmoid, **kwargs): super().__init__(**kwargs) self.input_dim = input_dim self.num_relation_types = num_relation_types - self.drop_prob = drop_prob + self.keep_prob = keep_prob self.activation = activation self.relation = [ @@ -95,8 +95,8 @@ class BilinearDecoder(torch.nn.Module): def forward(self, inputs_row, inputs_col): outputs = [] for k in range(self.num_relation_types): - inputs_row = dropout(inputs_row, 1.-self.drop_prob) - inputs_col = dropout(inputs_col, 1.-self.drop_prob) + inputs_row = dropout(inputs_row, self.keep_prob) + inputs_col = dropout(inputs_col, self.keep_prob) intermediate_product = torch.mm(inputs_row, self.relation[k]) rec = torch.bmm(intermediate_product.view(intermediate_product.shape[0], 1, intermediate_product.shape[1]), @@ -108,21 +108,21 @@ class BilinearDecoder(torch.nn.Module): class InnerProductDecoder(torch.nn.Module): """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" - def __init__(self, input_dim, num_relation_types, drop_prob=0., + def __init__(self, input_dim, num_relation_types, keep_prob=1., activation=torch.sigmoid, **kwargs): super().__init__(**kwargs) self.input_dim = input_dim self.num_relation_types = num_relation_types - self.drop_prob = drop_prob + self.keep_prob = keep_prob self.activation = activation def forward(self, inputs_row, inputs_col): outputs = [] for k in range(self.num_relation_types): - inputs_row = dropout(inputs_row, 1.-self.drop_prob) - inputs_col = dropout(inputs_col, 1.-self.drop_prob) + inputs_row = dropout(inputs_row, self.keep_prob) + inputs_col = dropout(inputs_col, self.keep_prob) rec = torch.bmm(inputs_row.view(inputs_row.shape[0], 1, inputs_row.shape[1]), inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) diff --git a/tests/icosagon/test_decode.py b/tests/icosagon/test_decode.py new file mode 100644 index 0000000..001f411 --- /dev/null +++ b/tests/icosagon/test_decode.py @@ -0,0 +1,86 @@ +from icosagon.decode import DEDICOMDecoder, \ + DistMultDecoder, \ + BilinearDecoder, \ + InnerProductDecoder +import decagon_pytorch.decode.pairwise +import torch + + +def test_dedicom_decoder_01(): + repr_ = torch.rand(20, 32) + dec_1 = DEDICOMDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + dec_2 = decagon_pytorch.decode.pairwise.DEDICOMDecoder(32, 7, drop_prob=0., + activation=torch.sigmoid) + dec_2.global_interaction = dec_1.global_interaction + dec_2.local_variation = dec_1.local_variation + + res_1 = dec_1(repr_, repr_) + res_2 = dec_2(repr_, repr_) + + assert isinstance(res_1, list) + assert isinstance(res_2, list) + + assert len(res_1) == len(res_2) + + for i in range(len(res_1)): + assert torch.all(res_1[i] == res_2[i]) + + +def test_dist_mult_decoder_01(): + repr_ = torch.rand(20, 32) + dec_1 = DistMultDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + dec_2 = decagon_pytorch.decode.pairwise.DistMultDecoder(32, 7, drop_prob=0., + activation=torch.sigmoid) + dec_2.relation = dec_1.relation + + res_1 = dec_1(repr_, repr_) + res_2 = dec_2(repr_, repr_) + + assert isinstance(res_1, list) + assert isinstance(res_2, list) + + assert len(res_1) == len(res_2) + + for i in range(len(res_1)): + assert torch.all(res_1[i] == res_2[i]) + + +def test_bilinear_decoder_01(): + repr_ = torch.rand(20, 32) + dec_1 = BilinearDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + dec_2 = decagon_pytorch.decode.pairwise.BilinearDecoder(32, 7, drop_prob=0., + activation=torch.sigmoid) + dec_2.relation = dec_1.relation + + res_1 = dec_1(repr_, repr_) + res_2 = dec_2(repr_, repr_) + + assert isinstance(res_1, list) + assert isinstance(res_2, list) + + assert len(res_1) == len(res_2) + + for i in range(len(res_1)): + assert torch.all(res_1[i] == res_2[i]) + + +def test_inner_product_decoder_01(): + repr_ = torch.rand(20, 32) + dec_1 = InnerProductDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + dec_2 = decagon_pytorch.decode.pairwise.InnerProductDecoder(32, 7, drop_prob=0., + activation=torch.sigmoid) + + res_1 = dec_1(repr_, repr_) + res_2 = dec_2(repr_, repr_) + + assert isinstance(res_1, list) + assert isinstance(res_2, list) + + assert len(res_1) == len(res_2) + + for i in range(len(res_1)): + assert torch.all(res_1[i] == res_2[i]) -- 2.26.2 From 702079c7e9ea219ddcc54b3f1f24b97c04886f6f Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sun, 7 Jun 2020 14:55:47 +0200 Subject: [PATCH 067/227] Add sampling tests. --- tests/icosagon/test_sampling.py | 170 ++++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 tests/icosagon/test_sampling.py diff --git a/tests/icosagon/test_sampling.py b/tests/icosagon/test_sampling.py new file mode 100644 index 0000000..3bc3327 --- /dev/null +++ b/tests/icosagon/test_sampling.py @@ -0,0 +1,170 @@ +import tensorflow as tf +import numpy as np +from collections import defaultdict +import torch +import torch.utils.data +from typing import List, \ + Union +import icosagon.sampling +import scipy.stats + + +def test_unigram_01(): + range_max = 7 + distortion = 0.75 + batch_size = 500 + unigrams = [ 1, 3, 2, 1, 2, 1, 3] + num_true = 1 + + true_classes = np.zeros((batch_size, num_true), dtype=np.int64) + for i in range(batch_size): + true_classes[i, 0] = i % range_max + true_classes = tf.convert_to_tensor(true_classes) + + neg_samples, _, _ = tf.nn.fixed_unigram_candidate_sampler( + true_classes=true_classes, + num_true=num_true, + num_sampled=batch_size, + unique=False, + range_max=range_max, + distortion=distortion, + unigrams=unigrams) + + assert neg_samples.shape == (batch_size,) + + for i in range(batch_size): + assert neg_samples[i] != true_classes[i, 0] + + counts = defaultdict(int) + with tf.Session() as sess: + neg_samples = neg_samples.eval() + for x in neg_samples: + counts[x] += 1 + + print('counts:', counts) + + assert counts[0] < counts[1] and \ + counts[0] < counts[2] and \ + counts[0] < counts[4] and \ + counts[0] < counts[6] + + assert counts[2] < counts[1] and \ + counts[0] < counts[6] + + assert counts[3] < counts[1] and \ + counts[3] < counts[2] and \ + counts[3] < counts[4] and \ + counts[3] < counts[6] + + assert counts[4] < counts[1] and \ + counts[4] < counts[6] + + assert counts[5] < counts[1] and \ + counts[5] < counts[2] and \ + counts[5] < counts[4] and \ + counts[5] < counts[6] + + +def test_unigram_02(): + range_max = 7 + distortion = 0.75 + batch_size = 500 + unigrams = [ 1, 3, 2, 1, 2, 1, 3] + num_true = 1 + + true_classes = np.zeros((batch_size, num_true), dtype=np.int64) + for i in range(batch_size): + true_classes[i, 0] = i % range_max + true_classes = torch.tensor(true_classes) + + neg_samples = icosagon.sampling.fixed_unigram_candidate_sampler( + true_classes=true_classes, + unigrams=unigrams, + distortion=distortion) + + assert neg_samples.shape == (batch_size,) + + for i in range(batch_size): + assert neg_samples[i] != true_classes[i, 0] + + counts = defaultdict(int) + for x in neg_samples: + counts[x.item()] += 1 + + print('counts:', counts) + + assert counts[0] < counts[1] and \ + counts[0] < counts[2] and \ + counts[0] < counts[4] and \ + counts[0] < counts[6] + + assert counts[2] < counts[1] and \ + counts[0] < counts[6] + + assert counts[3] < counts[1] and \ + counts[3] < counts[2] and \ + counts[3] < counts[4] and \ + counts[3] < counts[6] + + assert counts[4] < counts[1] and \ + counts[4] < counts[6] + + assert counts[5] < counts[1] and \ + counts[5] < counts[2] and \ + counts[5] < counts[4] and \ + counts[5] < counts[6] + + +def test_unigram_03(): + range_max = 7 + distortion = 0.75 + batch_size = 25 + unigrams = [ 1, 3, 2, 1, 2, 1, 3] + num_true = 1 + + true_classes = np.zeros((batch_size, num_true), dtype=np.int64) + for i in range(batch_size): + true_classes[i, 0] = i % range_max + + true_classes_tf = tf.convert_to_tensor(true_classes) + true_classes_torch = torch.tensor(true_classes) + + counts_tf = defaultdict(list) + counts_torch = defaultdict(list) + + for i in range(100): + neg_samples, _, _ = tf.nn.fixed_unigram_candidate_sampler( + true_classes=true_classes_tf, + num_true=num_true, + num_sampled=batch_size, + unique=False, + range_max=range_max, + distortion=distortion, + unigrams=unigrams) + + counts = defaultdict(int) + with tf.Session() as sess: + neg_samples = neg_samples.eval() + for x in neg_samples: + counts[x.item()] += 1 + for k, v in counts.items(): + counts_tf[k].append(v) + + neg_samples = icosagon.sampling.fixed_unigram_candidate_sampler( + true_classes=true_classes, + distortion=distortion, + unigrams=unigrams) + + counts = defaultdict(int) + for x in neg_samples: + counts[x.item()] += 1 + for k, v in counts.items(): + counts_torch[k].append(v) + + for i in range(range_max): + print('counts_tf[%d]:' % i, counts_tf[i]) + print('counts_torch[%d]:' % i, counts_torch[i]) + + for i in range(range_max): + statistic, pvalue = scipy.stats.ttest_ind(counts_tf[i], counts_torch[i]) + assert pvalue * range_max > .05 -- 2.26.2 From 6d0039b4479019e3504640ce67929d5e8b8303dc Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sun, 7 Jun 2020 15:12:26 +0200 Subject: [PATCH 068/227] Add test for init_glorot. --- tests/icosagon/test_weights.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/icosagon/test_weights.py diff --git a/tests/icosagon/test_weights.py b/tests/icosagon/test_weights.py new file mode 100644 index 0000000..7456076 --- /dev/null +++ b/tests/icosagon/test_weights.py @@ -0,0 +1,13 @@ +from icosagon.weights import init_glorot +import torch +import numpy as np + + +def test_init_glorot_01(): + torch.random.manual_seed(0) + res = init_glorot(10, 20) + torch.random.manual_seed(0) + rnd = torch.rand((10, 20)) + init_range = np.sqrt(6.0 / 30) + expected = -init_range + 2 * init_range * rnd + assert torch.all(res == expected) -- 2.26.2 From 1868014ca37f908e590195247853785d36a9c700 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sun, 7 Jun 2020 15:39:51 +0200 Subject: [PATCH 069/227] Work on declayer. --- src/icosagon/declayer.py | 105 +++++++++++++++++++++++++++++++- tests/icosagon/test_sampling.py | 2 +- tests/icosagon/test_weights.py | 10 +++ 3 files changed, 114 insertions(+), 3 deletions(-) diff --git a/src/icosagon/declayer.py b/src/icosagon/declayer.py index 46bffca..78ae8b7 100644 --- a/src/icosagon/declayer.py +++ b/src/icosagon/declayer.py @@ -1,2 +1,103 @@ -# from .layer import DecagonLayer -# from .input import OneHotInputLayer +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import torch +from .data import Data +from .trainprep import PreparedData, \ + TrainValTest +from typing import Type, \ + List, \ + Callable, \ + Union, \ + Dict, \ + Tuple +from .decode import DEDICOMDecoder + + +class DecodeLayer(torch.nn.Module): + def __init__(self, + input_dim: List[int], + data: Union[Data, PreparedData], + keep_prob: float = 1., + activation: Callable[[torch.Tensor], torch.Tensor] = torch.sigmoid, + decoder_class: Union[Type, Dict[Tuple[int, int], Type]] = DEDICOMDecoder, + **kwargs) -> None: + + super().__init__(**kwargs) + + assert all([ a == input_dim[0] \ + for a in input_dim ]) + + self.input_dim = input_dim + self.output_dim = 1 + self.data = data + self.keep_prob = keep_prob + self.activation = activation + + self.decoder_class = decoder_class + self.decoders = None + self.build() + + def build(self) -> None: + self.decoders = {} + + n = len(self.data.node_types) + for node_type_row in range(n): + if node_type_row not in relation_types: + continue + + for node_type_column in range(n): + if node_type_column not in relation_types[node_type_row]: + continue + + rels = relation_types[node_type_row][node_type_column] + if len(rels) == 0: + continue + + if isinstance(self.decoder_class, dict): + if (node_type_row, node_type_column) in self.decoder_class: + decoder_class = self.decoder_class[node_type_row, node_type_column] + elif (node_type_column, node_type_row) in self.decoder_class: + decoder_class = self.decoder_class[node_type_column, node_type_row] + else: + raise KeyError('Decoder not specified for edge type: %s -- %s' % ( + self.data.node_types[node_type_row].name, + self.data.node_types[node_type_column].name)) + else: + decoder_class = self.decoder_class + + self.decoders[node_type_row, node_type_column] = \ + decoder_class(self.input_dim, + num_relation_types = len(rels), + drop_prob = 1. - self.keep_prob, + activation = self.activation) + + def forward(self, last_layer_repr: List[torch.Tensor]) -> TrainValTest: + + # n = len(self.data.node_types) + # relation_types = self.data.relation_types + # for node_type_row in range(n): + # if node_type_row not in relation_types: + # continue + # + # for node_type_column in range(n): + # if node_type_column not in relation_types[node_type_row]: + # continue + # + # rels = relation_types[node_type_row][node_type_column] + # + # for mode in ['train', 'val', 'test']: + # getattr(relation_types[node_type_row][node_type_column].edges_pos, mode) + # getattr(self.data.edges_neg, mode) + # last_layer[] + + res = {} + for (node_type_row, node_type_column), dec in self.decoders.items(): + inputs_row = last_layer_repr[node_type_row] + inputs_column = last_layer_repr[node_type_column] + pred_adj_matrices = dec(inputs_row, inputs_col) + res[node_type_row, node_type_col] = pred_adj_matrices + return res diff --git a/tests/icosagon/test_sampling.py b/tests/icosagon/test_sampling.py index 3bc3327..8552949 100644 --- a/tests/icosagon/test_sampling.py +++ b/tests/icosagon/test_sampling.py @@ -132,7 +132,7 @@ def test_unigram_03(): counts_tf = defaultdict(list) counts_torch = defaultdict(list) - for i in range(100): + for i in range(10): neg_samples, _, _ = tf.nn.fixed_unigram_candidate_sampler( true_classes=true_classes_tf, num_true=num_true, diff --git a/tests/icosagon/test_weights.py b/tests/icosagon/test_weights.py index 7456076..5ddb997 100644 --- a/tests/icosagon/test_weights.py +++ b/tests/icosagon/test_weights.py @@ -11,3 +11,13 @@ def test_init_glorot_01(): init_range = np.sqrt(6.0 / 30) expected = -init_range + 2 * init_range * rnd assert torch.all(res == expected) + + +def test_init_glorot_02(): + torch.random.manual_seed(0) + res = init_glorot(20, 10) + torch.random.manual_seed(0) + rnd = torch.rand((20, 10)) + init_range = np.sqrt(6.0 / 30) + expected = -init_range + 2 * init_range * rnd + assert torch.all(res == expected) -- 2.26.2 From f51855e5b37f8853483dcaafc03bea2ea43350b3 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sun, 7 Jun 2020 15:54:10 +0200 Subject: [PATCH 070/227] Add first test for declayer. --- src/icosagon/declayer.py | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/src/icosagon/declayer.py b/src/icosagon/declayer.py index 78ae8b7..68dfdd1 100644 --- a/src/icosagon/declayer.py +++ b/src/icosagon/declayer.py @@ -22,8 +22,8 @@ class DecodeLayer(torch.nn.Module): input_dim: List[int], data: Union[Data, PreparedData], keep_prob: float = 1., - activation: Callable[[torch.Tensor], torch.Tensor] = torch.sigmoid, decoder_class: Union[Type, Dict[Tuple[int, int], Type]] = DEDICOMDecoder, + activation: Callable[[torch.Tensor], torch.Tensor] = torch.sigmoid, **kwargs) -> None: super().__init__(**kwargs) @@ -35,9 +35,9 @@ class DecodeLayer(torch.nn.Module): self.output_dim = 1 self.data = data self.keep_prob = keep_prob + self.decoder_class = decoder_class self.activation = activation - self.decoder_class = decoder_class self.decoders = None self.build() @@ -45,6 +45,7 @@ class DecodeLayer(torch.nn.Module): self.decoders = {} n = len(self.data.node_types) + relation_types = self.data.relation_types for node_type_row in range(n): if node_type_row not in relation_types: continue @@ -70,34 +71,16 @@ class DecodeLayer(torch.nn.Module): decoder_class = self.decoder_class self.decoders[node_type_row, node_type_column] = \ - decoder_class(self.input_dim, + decoder_class(self.input_dim[node_type_row], num_relation_types = len(rels), - drop_prob = 1. - self.keep_prob, + keep_prob = self.keep_prob, activation = self.activation) - def forward(self, last_layer_repr: List[torch.Tensor]) -> TrainValTest: - - # n = len(self.data.node_types) - # relation_types = self.data.relation_types - # for node_type_row in range(n): - # if node_type_row not in relation_types: - # continue - # - # for node_type_column in range(n): - # if node_type_column not in relation_types[node_type_row]: - # continue - # - # rels = relation_types[node_type_row][node_type_column] - # - # for mode in ['train', 'val', 'test']: - # getattr(relation_types[node_type_row][node_type_column].edges_pos, mode) - # getattr(self.data.edges_neg, mode) - # last_layer[] - + def forward(self, last_layer_repr: List[torch.Tensor]) -> Dict[Tuple[int, int], List[torch.Tensor]]: res = {} for (node_type_row, node_type_column), dec in self.decoders.items(): inputs_row = last_layer_repr[node_type_row] inputs_column = last_layer_repr[node_type_column] - pred_adj_matrices = dec(inputs_row, inputs_col) - res[node_type_row, node_type_col] = pred_adj_matrices + pred_adj_matrices = dec(inputs_row, inputs_column) + res[node_type_row, node_type_column] = pred_adj_matrices return res -- 2.26.2 From dc9263483017ff78fd062028ee538e12cb2c9296 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sun, 7 Jun 2020 15:55:59 +0200 Subject: [PATCH 071/227] Same test with torch.nn.Sequential. --- tests/icosagon/test_declayer.py | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/icosagon/test_declayer.py diff --git a/tests/icosagon/test_declayer.py b/tests/icosagon/test_declayer.py new file mode 100644 index 0000000..fb29831 --- /dev/null +++ b/tests/icosagon/test_declayer.py @@ -0,0 +1,50 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +from icosagon.input import OneHotInputLayer +from icosagon.convlayer import DecagonLayer +from icosagon.declayer import DecodeLayer +from icosagon.decode import DEDICOMDecoder +from icosagon.data import Data +import torch + + +def test_decode_layer_01(): + d = Data() + d.add_node_type('Dummy', 100) + d.add_relation_type('Dummy Relation 1', 0, 0, + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + in_layer = OneHotInputLayer(d) + d_layer = DecagonLayer(in_layer.output_dim, 32, d) + seq = torch.nn.Sequential(in_layer, d_layer) + last_layer_repr = seq(None) + dec = DecodeLayer(input_dim=d_layer.output_dim, data=d, keep_prob=1., + decoder_class=DEDICOMDecoder, activation=lambda x: x) + pred_adj_matrices = dec(last_layer_repr) + assert isinstance(pred_adj_matrices, dict) + assert len(pred_adj_matrices) == 1 + assert isinstance(pred_adj_matrices[0, 0], list) + assert len(pred_adj_matrices[0, 0]) == 1 + + +def test_decode_layer_02(): + d = Data() + d.add_node_type('Dummy', 100) + d.add_relation_type('Dummy Relation 1', 0, 0, + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + + in_layer = OneHotInputLayer(d) + d_layer = DecagonLayer(in_layer.output_dim, 32, d) + dec_layer = DecodeLayer(input_dim=d_layer.output_dim, data=d, keep_prob=1., + decoder_class=DEDICOMDecoder, activation=lambda x: x) + seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) + + pred_adj_matrices = seq(None) + + assert isinstance(pred_adj_matrices, dict) + assert len(pred_adj_matrices) == 1 + assert isinstance(pred_adj_matrices[0, 0], list) + assert len(pred_adj_matrices[0, 0]) == 1 -- 2.26.2 From bd894a02572531a43cceac114f8b7c2299052a2b Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sun, 7 Jun 2020 18:10:14 +0200 Subject: [PATCH 072/227] Move back to using single-level dictionary for Data.relation_types. --- src/icosagon/convlayer.py | 33 ++++++++---------------- src/icosagon/data.py | 38 +++++++++++---------------- src/icosagon/declayer.py | 44 ++++++++++++-------------------- src/icosagon/trainprep.py | 8 +++--- tests/icosagon/test_convlayer.py | 4 +-- 5 files changed, 49 insertions(+), 78 deletions(-) diff --git a/src/icosagon/convlayer.py b/src/icosagon/convlayer.py index 88f15b8..d356afc 100644 --- a/src/icosagon/convlayer.py +++ b/src/icosagon/convlayer.py @@ -51,34 +51,23 @@ class DecagonLayer(torch.nn.Module): self.build() def build(self): - n = len(self.data.node_types) - rel_types = self.data.relation_types - - self.next_layer_repr = [ [] for _ in range(n) ] + self.next_layer_repr = [ [] for _ in range(len(self.data.node_types)) ] - for node_type_row in range(n): - if node_type_row not in rel_types: + for (node_type_row, node_type_column), rels in self.data.relation_types.items(): + if len(rels) == 0: continue - for node_type_column in range(n): - if node_type_column not in rel_types[node_type_row]: - continue - - rels = rel_types[node_type_row][node_type_column] - if len(rels) == 0: - continue - - convolutions = [] + convolutions = [] - for r in rels: - conv = DropoutGraphConvActivation(self.input_dim[node_type_column], - self.output_dim[node_type_row], r.adjacency_matrix, - self.keep_prob, self.rel_activation) + for r in rels: + conv = DropoutGraphConvActivation(self.input_dim[node_type_column], + self.output_dim[node_type_row], r.adjacency_matrix, + self.keep_prob, self.rel_activation) - convolutions.append(conv) + convolutions.append(conv) - self.next_layer_repr[node_type_row].append( - Convolutions(node_type_column, convolutions)) + self.next_layer_repr[node_type_row].append( + Convolutions(node_type_column, convolutions)) def __call__(self, prev_layer_repr): next_layer_repr = [ [] for _ in range(len(self.data.node_types)) ] diff --git a/src/icosagon/data.py b/src/icosagon/data.py index 4166d40..b52dadc 100644 --- a/src/icosagon/data.py +++ b/src/icosagon/data.py @@ -7,6 +7,9 @@ from collections import defaultdict from dataclasses import dataclass import torch +from typing import List, \ + Dict, \ + Tuple @dataclass @@ -25,9 +28,12 @@ class RelationType(object): class Data(object): + node_types: List[NodeType] + relation_types: Dict[Tuple[int, int], List[RelationType]] + def __init__(self) -> None: self.node_types = [] - self.relation_types = defaultdict(lambda: defaultdict(list)) + self.relation_types = defaultdict(list) def add_node_type(self, name: str, count: int) -> None: name = str(name) @@ -73,14 +79,14 @@ class Data(object): adjacency_matrix_backward is not None: raise ValueError('Relation between nodes of the same type must be expressed using a single matrix') - self.relation_types[node_type_row][node_type_column].append( + self.relation_types[node_type_row, node_type_column].append( RelationType(name, node_type_row, node_type_column, adjacency_matrix, False)) if node_type_row != node_type_column and two_way: if adjacency_matrix_backward is None: adjacency_matrix_backward = adjacency_matrix.transpose(0, 1) - self.relation_types[node_type_column][node_type_row].append( + self.relation_types[node_type_column, node_type_row].append( RelationType(name, node_type_column, node_type_row, adjacency_matrix_backward, True)) @@ -99,16 +105,15 @@ class Data(object): s_1 = '' count = 0 - for i in range(n): - for j in range(n): - if i not in self.relation_types or \ - j not in self.relation_types[i]: + for node_type_row in range(n): + for node_type_column in range(n): + if (node_type_row, node_type_column) not in self.relation_types: continue - s_1 += ' - ' + self.node_types[i].name + ' -- ' + \ - self.node_types[j].name + ':\n' + s_1 += ' - ' + self.node_types[node_type_row].name + ' -- ' + \ + self.node_types[node_type_column].name + ':\n' - for r in self.relation_types[i][j]: + for r in self.relation_types[node_type_row, node_type_column]: if r.is_autogenerated: continue s_1 += ' - ' + r.name + '\n' @@ -118,16 +123,3 @@ class Data(object): s += s_1 return s.strip() - - # n = sum(map(len, self.relation_types)) - # - # for i in range(n): - # for j in range(n): - # key = (i, j) - # if key not in self.relation_types: - # continue - # rels = self.relation_types[key] - # - # for r in rels: - # - # return s.strip() diff --git a/src/icosagon/declayer.py b/src/icosagon/declayer.py index 68dfdd1..5b85c06 100644 --- a/src/icosagon/declayer.py +++ b/src/icosagon/declayer.py @@ -44,37 +44,27 @@ class DecodeLayer(torch.nn.Module): def build(self) -> None: self.decoders = {} - n = len(self.data.node_types) - relation_types = self.data.relation_types - for node_type_row in range(n): - if node_type_row not in relation_types: + for (node_type_row, node_type_column), rels in self.data.relation_types.items(): + if len(rels) == 0: continue - for node_type_column in range(n): - if node_type_column not in relation_types[node_type_row]: - continue - - rels = relation_types[node_type_row][node_type_column] - if len(rels) == 0: - continue - - if isinstance(self.decoder_class, dict): - if (node_type_row, node_type_column) in self.decoder_class: - decoder_class = self.decoder_class[node_type_row, node_type_column] - elif (node_type_column, node_type_row) in self.decoder_class: - decoder_class = self.decoder_class[node_type_column, node_type_row] - else: - raise KeyError('Decoder not specified for edge type: %s -- %s' % ( - self.data.node_types[node_type_row].name, - self.data.node_types[node_type_column].name)) + if isinstance(self.decoder_class, dict): + if (node_type_row, node_type_column) in self.decoder_class: + decoder_class = self.decoder_class[node_type_row, node_type_column] + elif (node_type_column, node_type_row) in self.decoder_class: + decoder_class = self.decoder_class[node_type_column, node_type_row] else: - decoder_class = self.decoder_class + raise KeyError('Decoder not specified for edge type: %s -- %s' % ( + self.data.node_types[node_type_row].name, + self.data.node_types[node_type_column].name)) + else: + decoder_class = self.decoder_class - self.decoders[node_type_row, node_type_column] = \ - decoder_class(self.input_dim[node_type_row], - num_relation_types = len(rels), - keep_prob = self.keep_prob, - activation = self.activation) + self.decoders[node_type_row, node_type_column] = \ + decoder_class(self.input_dim[node_type_row], + num_relation_types = len(rels), + keep_prob = self.keep_prob, + activation = self.activation) def forward(self, last_layer_repr: List[torch.Tensor]) -> Dict[Tuple[int, int], List[torch.Tensor]]: res = {} diff --git a/src/icosagon/trainprep.py b/src/icosagon/trainprep.py index 10886a1..a979290 100644 --- a/src/icosagon/trainprep.py +++ b/src/icosagon/trainprep.py @@ -46,7 +46,7 @@ class PreparedRelationType(object): @dataclass class PreparedData(object): node_types: List[NodeType] - relation_types: Dict[int, Dict[int, List[PreparedRelationType]]] + relation_types: Dict[Tuple[int, int], List[PreparedRelationType]] def train_val_test_split_edges(edges: torch.Tensor, @@ -137,9 +137,9 @@ def prepare_training(data: Data) -> PreparedData: if not isinstance(data, Data): raise ValueError('data must be of class Data') - relation_types = defaultdict(lambda: defaultdict(list)) - for (node_type_row, node_type_column), rels in data.relation_types: + relation_types = defaultdict(list) + for (node_type_row, node_type_column), rels in data.relation_types.items(): for r in rels: - relation_types[node_type_row][node_type_column].append( + relation_types[node_type_row, node_type_column].append( prep_relation_type(r)) return PreparedData(data.node_types, relation_types) diff --git a/tests/icosagon/test_convlayer.py b/tests/icosagon/test_convlayer.py index 82b7f56..8e713c2 100644 --- a/tests/icosagon/test_convlayer.py +++ b/tests/icosagon/test_convlayer.py @@ -86,7 +86,7 @@ def test_decagon_layer_04(): in_layer = OneHotInputLayer(d) multi_dgca = MultiDGCA([10], 32, - [r.adjacency_matrix for r in d.relation_types[0][0]], + [r.adjacency_matrix for r in d.relation_types[0, 0]], keep_prob=1., activation=lambda x: x) d_layer = DecagonLayer(in_layer.output_dim, 32, d, @@ -129,7 +129,7 @@ def test_decagon_layer_05(): in_layer = OneHotInputLayer(d) multi_dgca = MultiDGCA([100, 100], 32, - [r.adjacency_matrix for r in d.relation_types[0][0]], + [r.adjacency_matrix for r in d.relation_types[0, 0]], keep_prob=1., activation=lambda x: x) d_layer = DecagonLayer(in_layer.output_dim, output_dim=32, data=d, -- 2.26.2 From eda27bee0e90bccec2e8432bb499baa9329a800d Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sun, 7 Jun 2020 18:30:10 +0200 Subject: [PATCH 073/227] Increase test coverage. --- src/icosagon/convlayer.py | 7 ++++-- src/icosagon/input.py | 4 ++-- tests/icosagon/test_declayer.py | 41 ++++++++++++++++++++++++++++++++- tests/icosagon/test_input.py | 14 +++++++++++ 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/src/icosagon/convlayer.py b/src/icosagon/convlayer.py index d356afc..3fe1313 100644 --- a/src/icosagon/convlayer.py +++ b/src/icosagon/convlayer.py @@ -80,7 +80,10 @@ class DecagonLayer(torch.nn.Module): repr_ = sum(repr_) repr_ = torch.nn.functional.normalize(repr_, p=2, dim=1) next_layer_repr[node_type_row].append(repr_) - next_layer_repr[node_type_row] = sum(next_layer_repr[node_type_row]) - next_layer_repr[node_type_row] = self.layer_activation(next_layer_repr[node_type_row]) + if len(next_layer_repr[node_type_row]) == 0: + next_layer_repr[node_type_row] = torch.zeros(self.output_dim[node_type_row]) + else: + next_layer_repr[node_type_row] = sum(next_layer_repr[node_type_row]) + next_layer_repr[node_type_row] = self.layer_activation(next_layer_repr[node_type_row]) return next_layer_repr diff --git a/src/icosagon/input.py b/src/icosagon/input.py index 4f7dd73..8c686b1 100644 --- a/src/icosagon/input.py +++ b/src/icosagon/input.py @@ -16,7 +16,7 @@ class InputLayer(torch.nn.Module): output_dim = output_dim or \ list(map(lambda a: a.count, data.node_types)) - + if not isinstance(output_dim, list): output_dim = [output_dim,] * len(data.node_types) @@ -72,7 +72,7 @@ class OneHotInputLayer(torch.nn.Module): def __repr__(self) -> str: s = '' - s += 'One-hot Icosagon input layer\n' + s += 'Icosagon one-hot input layer\n' s += ' # of node types: %d\n' % len(self.data.node_types) for nt in self.data.node_types: s += ' - %s (%d)\n' % (nt.name, nt.count) diff --git a/tests/icosagon/test_declayer.py b/tests/icosagon/test_declayer.py index fb29831..2649759 100644 --- a/tests/icosagon/test_declayer.py +++ b/tests/icosagon/test_declayer.py @@ -43,8 +43,47 @@ def test_decode_layer_02(): seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) pred_adj_matrices = seq(None) - + assert isinstance(pred_adj_matrices, dict) assert len(pred_adj_matrices) == 1 assert isinstance(pred_adj_matrices[0, 0], list) assert len(pred_adj_matrices[0, 0]) == 1 + + +def test_decode_layer_03(): + d = Data() + d.add_node_type('Dummy 1', 100) + d.add_node_type('Dummy 2', 100) + d.add_relation_type('Dummy Relation 1', 0, 1, + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + + in_layer = OneHotInputLayer(d) + d_layer = DecagonLayer(in_layer.output_dim, 32, d) + dec_layer = DecodeLayer(input_dim=d_layer.output_dim, data=d, keep_prob=1., + decoder_class={(0, 1): DEDICOMDecoder}, activation=lambda x: x) + seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) + + pred_adj_matrices = seq(None) + assert isinstance(pred_adj_matrices, dict) + assert len(pred_adj_matrices) == 2 + assert isinstance(pred_adj_matrices[0, 1], list) + assert isinstance(pred_adj_matrices[1, 0], list) + assert len(pred_adj_matrices[0, 1]) == 1 + assert len(pred_adj_matrices[1, 0]) == 1 + + +def test_decode_layer_04(): + d = Data() + d.add_node_type('Dummy', 100) + assert len(d.relation_types[0, 0]) == 0 + + in_layer = OneHotInputLayer(d) + d_layer = DecagonLayer(in_layer.output_dim, 32, d) + dec_layer = DecodeLayer(input_dim=d_layer.output_dim, data=d, keep_prob=1., + decoder_class=DEDICOMDecoder, activation=lambda x: x) + seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) + + pred_adj_matrices = seq(None) + + assert isinstance(pred_adj_matrices, dict) + assert len(pred_adj_matrices) == 0 diff --git a/tests/icosagon/test_input.py b/tests/icosagon/test_input.py index 3e73ce1..1e9cfbf 100644 --- a/tests/icosagon/test_input.py +++ b/tests/icosagon/test_input.py @@ -70,6 +70,13 @@ def test_input_layer_03(): assert layer.node_reps[1].device == device +def test_input_layer_04(): + d = _some_data() + layer = InputLayer(d, 32) + s = repr(layer) + assert s.startswith('Icosagon input layer') + + def test_one_hot_input_layer_01(): d = _some_data() layer = OneHotInputLayer(d) @@ -104,3 +111,10 @@ def test_one_hot_input_layer_03(): # assert layer.device.type == 'cuda:0' assert layer.node_reps[0].device == device assert layer.node_reps[1].device == device + + +def test_one_hot_input_layer_04(): + d = _some_data() + layer = OneHotInputLayer(d) + s = repr(layer) + assert s.startswith('Icosagon one-hot input layer') -- 2.26.2 From 53cd182e6ba2f0d4d42203bdb035a52ca802cf24 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 8 Jun 2020 10:44:08 +0200 Subject: [PATCH 074/227] Modify decoders to handle one relation at a time. --- src/icosagon/decode.py | 86 ++++++++++++++++------------------- tests/icosagon/test_decode.py | 8 ++-- 2 files changed, 43 insertions(+), 51 deletions(-) diff --git a/src/icosagon/decode.py b/src/icosagon/decode.py index dccf508..b69ec97 100644 --- a/src/icosagon/decode.py +++ b/src/icosagon/decode.py @@ -26,22 +26,20 @@ class DEDICOMDecoder(torch.nn.Module): for _ in range(num_relation_types) ] - def forward(self, inputs_row, inputs_col): - outputs = [] - for k in range(self.num_relation_types): - inputs_row = dropout(inputs_row, self.keep_prob) - inputs_col = dropout(inputs_col, self.keep_prob) + def forward(self, inputs_row, inputs_col, relation_index): + inputs_row = dropout(inputs_row, self.keep_prob) + inputs_col = dropout(inputs_col, self.keep_prob) - relation = torch.diag(self.local_variation[k]) + relation = torch.diag(self.local_variation[relation_index]) - product1 = torch.mm(inputs_row, relation) - product2 = torch.mm(product1, self.global_interaction) - product3 = torch.mm(product2, relation) - rec = torch.bmm(product3.view(product3.shape[0], 1, product3.shape[1]), - inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) - rec = torch.flatten(rec) - outputs.append(self.activation(rec)) - return outputs + product1 = torch.mm(inputs_row, relation) + product2 = torch.mm(product1, self.global_interaction) + product3 = torch.mm(product2, relation) + rec = torch.bmm(product3.view(product3.shape[0], 1, product3.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) + + return self.activation(rec) class DistMultDecoder(torch.nn.Module): @@ -60,20 +58,18 @@ class DistMultDecoder(torch.nn.Module): for _ in range(num_relation_types) ] - def forward(self, inputs_row, inputs_col): - outputs = [] - for k in range(self.num_relation_types): - inputs_row = dropout(inputs_row, self.keep_prob) - inputs_col = dropout(inputs_col, self.keep_prob) + def forward(self, inputs_row, inputs_col, relation_index): + inputs_row = dropout(inputs_row, self.keep_prob) + inputs_col = dropout(inputs_col, self.keep_prob) + + relation = torch.diag(self.relation[relation_index]) - relation = torch.diag(self.relation[k]) + intermediate_product = torch.mm(inputs_row, relation) + rec = torch.bmm(intermediate_product.view(intermediate_product.shape[0], 1, intermediate_product.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) - intermediate_product = torch.mm(inputs_row, relation) - rec = torch.bmm(intermediate_product.view(intermediate_product.shape[0], 1, intermediate_product.shape[1]), - inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) - rec = torch.flatten(rec) - outputs.append(self.activation(rec)) - return outputs + return self.activation(rec) class BilinearDecoder(torch.nn.Module): @@ -92,18 +88,16 @@ class BilinearDecoder(torch.nn.Module): for _ in range(num_relation_types) ] - def forward(self, inputs_row, inputs_col): - outputs = [] - for k in range(self.num_relation_types): - inputs_row = dropout(inputs_row, self.keep_prob) - inputs_col = dropout(inputs_col, self.keep_prob) + def forward(self, inputs_row, inputs_col, relation_index): + inputs_row = dropout(inputs_row, self.keep_prob) + inputs_col = dropout(inputs_col, self.keep_prob) - intermediate_product = torch.mm(inputs_row, self.relation[k]) - rec = torch.bmm(intermediate_product.view(intermediate_product.shape[0], 1, intermediate_product.shape[1]), - inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) - rec = torch.flatten(rec) - outputs.append(self.activation(rec)) - return outputs + intermediate_product = torch.mm(inputs_row, self.relation[relation_index]) + rec = torch.bmm(intermediate_product.view(intermediate_product.shape[0], 1, intermediate_product.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) + + return self.activation(rec) class InnerProductDecoder(torch.nn.Module): @@ -118,14 +112,12 @@ class InnerProductDecoder(torch.nn.Module): self.activation = activation - def forward(self, inputs_row, inputs_col): - outputs = [] - for k in range(self.num_relation_types): - inputs_row = dropout(inputs_row, self.keep_prob) - inputs_col = dropout(inputs_col, self.keep_prob) + def forward(self, inputs_row, inputs_col, _): + inputs_row = dropout(inputs_row, self.keep_prob) + inputs_col = dropout(inputs_col, self.keep_prob) + + rec = torch.bmm(inputs_row.view(inputs_row.shape[0], 1, inputs_row.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) - rec = torch.bmm(inputs_row.view(inputs_row.shape[0], 1, inputs_row.shape[1]), - inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) - rec = torch.flatten(rec) - outputs.append(self.activation(rec)) - return outputs + return self.activation(rec) diff --git a/tests/icosagon/test_decode.py b/tests/icosagon/test_decode.py index 001f411..c0ec4c3 100644 --- a/tests/icosagon/test_decode.py +++ b/tests/icosagon/test_decode.py @@ -15,7 +15,7 @@ def test_dedicom_decoder_01(): dec_2.global_interaction = dec_1.global_interaction dec_2.local_variation = dec_1.local_variation - res_1 = dec_1(repr_, repr_) + res_1 = [ dec_1(repr_, repr_, k) for k in range(7) ] res_2 = dec_2(repr_, repr_) assert isinstance(res_1, list) @@ -35,7 +35,7 @@ def test_dist_mult_decoder_01(): activation=torch.sigmoid) dec_2.relation = dec_1.relation - res_1 = dec_1(repr_, repr_) + res_1 = [ dec_1(repr_, repr_, k) for k in range(7) ] res_2 = dec_2(repr_, repr_) assert isinstance(res_1, list) @@ -55,7 +55,7 @@ def test_bilinear_decoder_01(): activation=torch.sigmoid) dec_2.relation = dec_1.relation - res_1 = dec_1(repr_, repr_) + res_1 = [ dec_1(repr_, repr_, k) for k in range(7) ] res_2 = dec_2(repr_, repr_) assert isinstance(res_1, list) @@ -74,7 +74,7 @@ def test_inner_product_decoder_01(): dec_2 = decagon_pytorch.decode.pairwise.InnerProductDecoder(32, 7, drop_prob=0., activation=torch.sigmoid) - res_1 = dec_1(repr_, repr_) + res_1 = [ dec_1(repr_, repr_, k) for k in range(7) ] res_2 = dec_2(repr_, repr_) assert isinstance(res_1, list) -- 2.26.2 From 15e95bdc727b5710888add2d752000fd7d3da810 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 8 Jun 2020 10:54:32 +0200 Subject: [PATCH 075/227] Use hints instead of is_autogenerated. --- src/icosagon/data.py | 15 +++++++++------ src/icosagon/declayer.py | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/icosagon/data.py b/src/icosagon/data.py index b52dadc..86b17d1 100644 --- a/src/icosagon/data.py +++ b/src/icosagon/data.py @@ -5,11 +5,12 @@ from collections import defaultdict -from dataclasses import dataclass +from dataclasses import dataclass, field import torch from typing import List, \ Dict, \ - Tuple + Tuple, \ + Any @dataclass @@ -24,7 +25,7 @@ class RelationType(object): node_type_row: int node_type_column: int adjacency_matrix: torch.Tensor - is_autogenerated: bool = False + hints: Dict[str, Any] = field(default_factory=dict) class Data(object): @@ -81,14 +82,16 @@ class Data(object): self.relation_types[node_type_row, node_type_column].append( RelationType(name, node_type_row, node_type_column, - adjacency_matrix, False)) + adjacency_matrix)) if node_type_row != node_type_column and two_way: + hints = { 'display': False } if adjacency_matrix_backward is None: adjacency_matrix_backward = adjacency_matrix.transpose(0, 1) + hints['symmetric'] = True self.relation_types[node_type_column, node_type_row].append( RelationType(name, node_type_column, node_type_row, - adjacency_matrix_backward, True)) + adjacency_matrix_backward, hints)) def __repr__(self): n = len(self.node_types) @@ -114,7 +117,7 @@ class Data(object): self.node_types[node_type_column].name + ':\n' for r in self.relation_types[node_type_row, node_type_column]: - if r.is_autogenerated: + if not r.hints.get('display', True): continue s_1 += ' - ' + r.name + '\n' count += 1 diff --git a/src/icosagon/declayer.py b/src/icosagon/declayer.py index 5b85c06..f5ed5b1 100644 --- a/src/icosagon/declayer.py +++ b/src/icosagon/declayer.py @@ -71,6 +71,6 @@ class DecodeLayer(torch.nn.Module): for (node_type_row, node_type_column), dec in self.decoders.items(): inputs_row = last_layer_repr[node_type_row] inputs_column = last_layer_repr[node_type_column] - pred_adj_matrices = dec(inputs_row, inputs_column) + pred_adj_matrices = [ dec(inputs_row, inputs_column, k) for k in range(dec.num_relation_types) ] res[node_type_row, node_type_column] = pred_adj_matrices return res -- 2.26.2 From d06e65b0bd0d447561cf326fa349dd47a0343b56 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 8 Jun 2020 12:34:46 +0200 Subject: [PATCH 076/227] Add icosagon classes diagram. --- .gitignore | 1 + docs/icosagon-classes.svg | 996 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 997 insertions(+) create mode 100644 docs/icosagon-classes.svg diff --git a/.gitignore b/.gitignore index 837e16b..9fd0d18 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ __pycache__ .cache/ +.coverage diff --git a/docs/icosagon-classes.svg b/docs/icosagon-classes.svg new file mode 100644 index 0000000..fea7692 --- /dev/null +++ b/docs/icosagon-classes.svg @@ -0,0 +1,996 @@ + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + Data + + + PreparedData + + + OneHotInputLayer + + + DecagonLayer + + + DecagonLayer + + + DecodeLayer + + + Loss + edges_pos + train + + val + test + train val + test + edges_neg + train + val + test + + + Metrics + + val + test + train + + + + + + + edges_pos + train + val + test + edges_neg + train + val + test + + edges_pos + train + val + test + edges_neg + train + val + test + + ... relations + -- 2.26.2 From 56ce7aa60b980ec768e33f443a73ab5f41a8159f Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 9 Jun 2020 13:07:08 +0200 Subject: [PATCH 077/227] Add type checks to DecodeLayer. --- src/icosagon/declayer.py | 20 +++++++++++++++++--- src/icosagon/trainprep.py | 4 ++-- tests/icosagon/test_declayer.py | 15 +++++++++++---- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/icosagon/declayer.py b/src/icosagon/declayer.py index f5ed5b1..9047025 100644 --- a/src/icosagon/declayer.py +++ b/src/icosagon/declayer.py @@ -20,7 +20,7 @@ from .decode import DEDICOMDecoder class DecodeLayer(torch.nn.Module): def __init__(self, input_dim: List[int], - data: Union[Data, PreparedData], + data: PreparedData, keep_prob: float = 1., decoder_class: Union[Type, Dict[Tuple[int, int], Type]] = DEDICOMDecoder, activation: Callable[[torch.Tensor], torch.Tensor] = torch.sigmoid, @@ -28,8 +28,22 @@ class DecodeLayer(torch.nn.Module): super().__init__(**kwargs) - assert all([ a == input_dim[0] \ - for a in input_dim ]) + if not isinstance(input_dim, list): + raise TypeError('input_dim must be a List') + + if not all([ a == input_dim[0] for a in input_dim ]): + raise ValueError('All elements of input_dim must have the same value') + + if not isinstance(data, PreparedData): + raise TypeError('data must be an instance of PreparedData') + + if not isinstance(decoder_class, type) and \ + not isinstance(decoder_class, dict): + raise TypeError('decoder_class must be a Type or a Dict') + + if not isinstance(decoder_class, dict): + decoder_class = { k: decoder_class \ + for k in data.relation_types.keys() } self.input_dim = input_dim self.output_dim = 1 diff --git a/src/icosagon/trainprep.py b/src/icosagon/trainprep.py index a979290..a505d9b 100644 --- a/src/icosagon/trainprep.py +++ b/src/icosagon/trainprep.py @@ -133,7 +133,7 @@ def prepare_relation_type(r: RelationType, adj_mat_train, edges_pos, edges_neg) -def prepare_training(data: Data) -> PreparedData: +def prepare_training(data: Data, ratios: TrainValTest) -> PreparedData: if not isinstance(data, Data): raise ValueError('data must be of class Data') @@ -141,5 +141,5 @@ def prepare_training(data: Data) -> PreparedData: for (node_type_row, node_type_column), rels in data.relation_types.items(): for r in rels: relation_types[node_type_row, node_type_column].append( - prep_relation_type(r)) + prepare_relation_type(r, ratios)) return PreparedData(data.node_types, relation_types) diff --git a/tests/icosagon/test_declayer.py b/tests/icosagon/test_declayer.py index 2649759..3536387 100644 --- a/tests/icosagon/test_declayer.py +++ b/tests/icosagon/test_declayer.py @@ -9,6 +9,8 @@ from icosagon.convlayer import DecagonLayer from icosagon.declayer import DecodeLayer from icosagon.decode import DEDICOMDecoder from icosagon.data import Data +from icosagon.trainprep import prepare_training, \ + TrainValTest import torch @@ -17,11 +19,12 @@ def test_decode_layer_01(): d.add_node_type('Dummy', 100) d.add_relation_type('Dummy Relation 1', 0, 0, torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) in_layer = OneHotInputLayer(d) d_layer = DecagonLayer(in_layer.output_dim, 32, d) seq = torch.nn.Sequential(in_layer, d_layer) last_layer_repr = seq(None) - dec = DecodeLayer(input_dim=d_layer.output_dim, data=d, keep_prob=1., + dec = DecodeLayer(input_dim=d_layer.output_dim, data=prep_d, keep_prob=1., decoder_class=DEDICOMDecoder, activation=lambda x: x) pred_adj_matrices = dec(last_layer_repr) assert isinstance(pred_adj_matrices, dict) @@ -35,10 +38,11 @@ def test_decode_layer_02(): d.add_node_type('Dummy', 100) d.add_relation_type('Dummy Relation 1', 0, 0, torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) in_layer = OneHotInputLayer(d) d_layer = DecagonLayer(in_layer.output_dim, 32, d) - dec_layer = DecodeLayer(input_dim=d_layer.output_dim, data=d, keep_prob=1., + dec_layer = DecodeLayer(input_dim=d_layer.output_dim, data=prep_d, keep_prob=1., decoder_class=DEDICOMDecoder, activation=lambda x: x) seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) @@ -56,10 +60,11 @@ def test_decode_layer_03(): d.add_node_type('Dummy 2', 100) d.add_relation_type('Dummy Relation 1', 0, 1, torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) in_layer = OneHotInputLayer(d) d_layer = DecagonLayer(in_layer.output_dim, 32, d) - dec_layer = DecodeLayer(input_dim=d_layer.output_dim, data=d, keep_prob=1., + dec_layer = DecodeLayer(input_dim=d_layer.output_dim, data=prep_d, keep_prob=1., decoder_class={(0, 1): DEDICOMDecoder}, activation=lambda x: x) seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) @@ -77,9 +82,11 @@ def test_decode_layer_04(): d.add_node_type('Dummy', 100) assert len(d.relation_types[0, 0]) == 0 + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) + in_layer = OneHotInputLayer(d) d_layer = DecagonLayer(in_layer.output_dim, 32, d) - dec_layer = DecodeLayer(input_dim=d_layer.output_dim, data=d, keep_prob=1., + dec_layer = DecodeLayer(input_dim=d_layer.output_dim, data=prep_d, keep_prob=1., decoder_class=DEDICOMDecoder, activation=lambda x: x) seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) -- 2.26.2 From 4b0f68d1ad2b739d9c2634f20d99a8a360c2eddd Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 9 Jun 2020 13:34:58 +0200 Subject: [PATCH 078/227] Add symmetry tests for decode. --- tests/icosagon/test_decode.py | 75 +++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/tests/icosagon/test_decode.py b/tests/icosagon/test_decode.py index c0ec4c3..ee0eaf2 100644 --- a/tests/icosagon/test_decode.py +++ b/tests/icosagon/test_decode.py @@ -84,3 +84,78 @@ def test_inner_product_decoder_01(): for i in range(len(res_1)): assert torch.all(res_1[i] == res_2[i]) + + +def test_is_dedicom_not_symmetric_01(): + repr_1 = torch.rand(20, 32) + repr_2 = torch.rand(20, 32) + dec = DEDICOMDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + + res_1 = [ dec(repr_1, repr_2, k) for k in range(7) ] + res_2 = [ dec(repr_2, repr_1, k) for k in range(7) ] + + + assert isinstance(res_1, list) + assert isinstance(res_2, list) + + assert len(res_1) == len(res_2) + + for i in range(len(res_1)): + assert not torch.all(res_1[i] - res_2[i] < 0.000001) + + +def test_is_dist_mult_symmetric_01(): + repr_1 = torch.rand(20, 32) + repr_2 = torch.rand(20, 32) + dec = DistMultDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + + res_1 = [ dec(repr_1, repr_2, k) for k in range(7) ] + res_2 = [ dec(repr_2, repr_1, k) for k in range(7) ] + + + assert isinstance(res_1, list) + assert isinstance(res_2, list) + + assert len(res_1) == len(res_2) + + for i in range(len(res_1)): + assert torch.all(res_1[i] - res_2[i] < 0.000001) + + +def test_is_bilinear_not_symmetric_01(): + repr_1 = torch.rand(20, 32) + repr_2 = torch.rand(20, 32) + dec = BilinearDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + + res_1 = [ dec(repr_1, repr_2, k) for k in range(7) ] + res_2 = [ dec(repr_2, repr_1, k) for k in range(7) ] + + assert isinstance(res_1, list) + assert isinstance(res_2, list) + + assert len(res_1) == len(res_2) + + for i in range(len(res_1)): + assert not torch.all(res_1[i] - res_2[i] < 0.000001) + + +def test_is_inner_product_symmetric_01(): + repr_1 = torch.rand(20, 32) + repr_2 = torch.rand(20, 32) + dec = InnerProductDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + + res_1 = [ dec(repr_1, repr_2, k) for k in range(7) ] + res_2 = [ dec(repr_2, repr_1, k) for k in range(7) ] + + + assert isinstance(res_1, list) + assert isinstance(res_2, list) + + assert len(res_1) == len(res_2) + + for i in range(len(res_1)): + assert torch.all(res_1[i] - res_2[i] < 0.000001) -- 2.26.2 From b5d2f8fcda9288394504427477eb6514310c73a0 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 9 Jun 2020 15:23:43 +0200 Subject: [PATCH 079/227] Start implementing RelationFamily. --- src/icosagon/data.py | 161 +++++++++++++++++++++++++----------- tests/icosagon/test_data.py | 33 +++++--- 2 files changed, 133 insertions(+), 61 deletions(-) diff --git a/src/icosagon/data.py b/src/icosagon/data.py index 86b17d1..cd55b29 100644 --- a/src/icosagon/data.py +++ b/src/icosagon/data.py @@ -10,7 +10,10 @@ import torch from typing import List, \ Dict, \ Tuple, \ - Any + Any, \ + Type +from .decode import DEDICOMDecoder, \ + BilinearDecoder @dataclass @@ -25,25 +28,33 @@ class RelationType(object): node_type_row: int node_type_column: int adjacency_matrix: torch.Tensor + two_way: bool hints: Dict[str, Any] = field(default_factory=dict) -class Data(object): - node_types: List[NodeType] - relation_types: Dict[Tuple[int, int], List[RelationType]] +class RelationFamily(object): + def __init__(self, + data: 'Data', + name: str, + node_type_row: int, + node_type_column: int, + is_symmetric: bool, + decoder_class: Type) -> None: - def __init__(self) -> None: - self.node_types = [] - self.relation_types = defaultdict(list) + if not is_symmetric and \ + decoder_class != DEDICOMDecoder and \ + decoder_class != BilinearDecoder: + raise TypeError('Family is assymetric but the specified decoder_class supports symmetric relations only') - def add_node_type(self, name: str, count: int) -> None: - name = str(name) - count = int(count) - if not name: - raise ValueError('You must provide a non-empty node type name') - if count <= 0: - raise ValueError('You must provide a positive node count') - self.node_types.append(NodeType(name, count)) + self.data = data + self.name = name + self.node_type_row = node_type_row + self.node_type_column = node_type_column + self.is_symmetric = is_symmetric + self.decoder_class = decoder_class + + self.relation_types = { (node_type_row, node_type_column): [], + (node_type_column, node_type_row): [] } def add_relation_type(self, name: str, node_type_row: int, node_type_column: int, adjacency_matrix: torch.Tensor, adjacency_matrix_backward: torch.Tensor = None, @@ -53,45 +64,110 @@ class Data(object): node_type_row = int(node_type_row) node_type_column = int(node_type_column) - if node_type_row < 0 or node_type_row > len(self.node_types): + if (node_type_row, node_type_column) not in self.relation_types: + raise ValueError('Specified node_type_row/node_type_column tuple does not belong to this family') + + if node_type_row < 0 or node_type_row >= len(self.data.node_types): raise ValueError('node_type_row outside of the valid range of node types') - if node_type_column < 0 or node_type_column > len(self.node_types): + if node_type_column < 0 or node_type_column >= len(self.data.node_types): raise ValueError('node_type_column outside of the valid range of node types') if not isinstance(adjacency_matrix, torch.Tensor): raise ValueError('adjacency_matrix must be a torch.Tensor') - if adjacency_matrix_backward and not isinstance(adjacency_matrix_backward, torch.Tensor): + if adjacency_matrix_backward is not None \ + and not isinstance(adjacency_matrix_backward, torch.Tensor): raise ValueError('adjacency_matrix_backward must be a torch.Tensor') - if adjacency_matrix.shape != (self.node_types[node_type_row].count, - self.node_types[node_type_column].count): + if adjacency_matrix.shape != (self.data.node_types[node_type_row].count, + self.data.node_types[node_type_column].count): raise ValueError('adjacency_matrix shape must be (num_row_nodes, num_column_nodes)') if adjacency_matrix_backward is not None and \ - adjacency_matrix_backward.shape != (self.node_types[node_type_column].count, - self.node_types[node_type_row].count): + adjacency_matrix_backward.shape != (self.data.node_types[node_type_column].count, + self.data.node_types[node_type_row].count): raise ValueError('adjacency_matrix shape must be (num_column_nodes, num_row_nodes)') - two_way = bool(two_way) - if node_type_row == node_type_column and \ adjacency_matrix_backward is not None: raise ValueError('Relation between nodes of the same type must be expressed using a single matrix') - self.relation_types[node_type_row, node_type_column].append( - RelationType(name, node_type_row, node_type_column, - adjacency_matrix)) + if self.is_symmetric and adjacency_matrix_backward is not None: + raise ValueError('Cannot use a custom adjacency_matrix_backward in a symmetric relation family') + + if self.is_symmetric and node_type_row == node_type_column and \ + not torch.all(adjacency_matrix == adjacency_matrix.transpose(0, 1)): + raise ValueError('Relation family is symmetric but adjacency_matrix is assymetric') + + two_way = bool(two_way) if node_type_row != node_type_column and two_way: - hints = { 'display': False } + print('%d != %d' % (node_type_row, node_type_column)) if adjacency_matrix_backward is None: adjacency_matrix_backward = adjacency_matrix.transpose(0, 1) - hints['symmetric'] = True self.relation_types[node_type_column, node_type_row].append( RelationType(name, node_type_column, node_type_row, - adjacency_matrix_backward, hints)) + adjacency_matrix_backward, two_way, { 'display': False })) + + self.relation_types[node_type_row, node_type_column].append( + RelationType(name, node_type_row, node_type_column, + adjacency_matrix, two_way)) + + def node_name(self, index): + return self.data.node_types[index].name + + def __repr__(self): + s = 'Relation family %s' % self.name + + for (node_type_row, node_type_column), rels in self.relation_types.items(): + for r in rels: + if 'display' in r.hints and not r.hints['display']: + continue + s += '\n - %s%s' % (r.name, ' (two-way)' if r.two_way else '%s <- %s' % \ + (self.node_name(node_type_row), self.node_name(node_type_column))) + + return s + + def repr_indented(self): + s = ' - %s' % self.name + + for (node_type_row, node_type_column), rels in self.relation_types.items(): + for r in rels: + if 'display' in r.hints and not r.hints['display']: + continue + s += '\n - %s%s' % (r.name, ' (two-way)' if r.two_way else '%s <- %s' % \ + (self.node_name(node_type_row), self.node_name(node_type_column))) + + return s + + +class Data(object): + node_types: List[NodeType] + relation_types: Dict[Tuple[int, int], List[RelationType]] + + def __init__(self) -> None: + self.node_types = [] + self.relation_families = [] + + def add_node_type(self, name: str, count: int) -> None: + name = str(name) + count = int(count) + if not name: + raise ValueError('You must provide a non-empty node type name') + if count <= 0: + raise ValueError('You must provide a positive node count') + self.node_types.append(NodeType(name, count)) + + def add_relation_family(self, name: str, node_type_row: int, + node_type_column: int, is_symmetric: bool, + decoder_class: Type = DEDICOMDecoder): + + fam = RelationFamily(self, name, node_type_row, node_type_column, + is_symmetric, decoder_class) + self.relation_families.append(fam) + + return fam def __repr__(self): n = len(self.node_types) @@ -102,27 +178,12 @@ class Data(object): s += '- ' + str(n) + ' node type(s):\n' for nt in self.node_types: s += ' - ' + nt.name + '\n' - if len(self.relation_types) == 0: - s += '- No relation types\n' + if len(self.relation_families) == 0: + s += '- No relation families\n' return s.strip() - s_1 = '' - count = 0 - for node_type_row in range(n): - for node_type_column in range(n): - if (node_type_row, node_type_column) not in self.relation_types: - continue - - s_1 += ' - ' + self.node_types[node_type_row].name + ' -- ' + \ - self.node_types[node_type_column].name + ':\n' - - for r in self.relation_types[node_type_row, node_type_column]: - if not r.hints.get('display', True): - continue - s_1 += ' - ' + r.name + '\n' - count += 1 - - s += '- %d relation type(s):\n' % count - s += s_1 + s += '- %d relation families:\n' % len(self.relation_families) + for fam in self.relation_families: + s += fam.repr_indented() + '\n' return s.strip() diff --git a/tests/icosagon/test_data.py b/tests/icosagon/test_data.py index 4ec8164..d67c1c1 100644 --- a/tests/icosagon/test_data.py +++ b/tests/icosagon/test_data.py @@ -17,11 +17,14 @@ def test_data_01(): dummy_1 = torch.zeros((1000, 100)) dummy_2 = torch.zeros((100, 100)) dummy_3 = torch.zeros((1000, 1000)) - d.add_relation_type('Target', 1, 0, dummy_0) - d.add_relation_type('Interaction', 0, 0, dummy_3) - d.add_relation_type('Side Effect: Nausea', 1, 1, dummy_2) - d.add_relation_type('Side Effect: Infertility', 1, 1, dummy_2) - d.add_relation_type('Side Effect: Death', 1, 1, dummy_2) + fam = d.add_relation_family('Drug-Gene', 1, 0, True) + fam.add_relation_type('Target', 1, 0, dummy_0) + fam = d.add_relation_family('Gene-Gene', 0, 0, True) + fam.add_relation_type('Interaction', 0, 0, dummy_3) + fam = d.add_relation_family('Drug-Drug', 1, 1, True) + fam.add_relation_type('Side Effect: Nausea', 1, 1, dummy_2) + fam.add_relation_type('Side Effect: Infertility', 1, 1, dummy_2) + fam.add_relation_type('Side Effect: Death', 1, 1, dummy_2) print(d) @@ -29,20 +32,27 @@ def test_data_02(): d = Data() d.add_node_type('Gene', 1000) d.add_node_type('Drug', 100) + dummy_0 = torch.zeros((100, 1000)) dummy_1 = torch.zeros((1000, 100)) dummy_2 = torch.zeros((100, 100)) dummy_3 = torch.zeros((1000, 1000)) + + fam = d.add_relation_family('Drug-Gene', 1, 0, True) with pytest.raises(ValueError): - d.add_relation_type('Target', 1, 0, dummy_1) + fam.add_relation_type('Target', 1, 0, dummy_1) + + fam = d.add_relation_family('Gene-Gene', 0, 0, True) with pytest.raises(ValueError): - d.add_relation_type('Interaction', 0, 0, dummy_2) + fam.add_relation_type('Interaction', 0, 0, dummy_2) + + fam = d.add_relation_family('Drug-Drug', 1, 1, True) with pytest.raises(ValueError): - d.add_relation_type('Side Effect: Nausea', 1, 1, dummy_3) + fam.add_relation_type('Side Effect: Nausea', 1, 1, dummy_3) with pytest.raises(ValueError): - d.add_relation_type('Side Effect: Infertility', 1, 1, dummy_3) + fam.add_relation_type('Side Effect: Infertility', 1, 1, dummy_3) with pytest.raises(ValueError): - d.add_relation_type('Side Effect: Death', 1, 1, dummy_3) + fam.add_relation_type('Side Effect: Death', 1, 1, dummy_3) print(d) @@ -50,6 +60,7 @@ def test_data_03(): d = Data() d.add_node_type('Gene', 1000) d.add_node_type('Drug', 100) + fam = d.add_relation_family('Drug-Gene', 1, 0, True) with pytest.raises(ValueError): - d.add_relation_type('Target', 1, 0, None) + fam.add_relation_type('Target', 1, 0, None) print(d) -- 2.26.2 From a1b0272553846e7dce3d4923265caf3d47cdc601 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 9 Jun 2020 19:37:45 +0200 Subject: [PATCH 080/227] Add icosagon-hierarchy.svg. --- docs/icosagon-hierarchy.svg | 334 ++++++++++++++++++++++++++++++++++++ 1 file changed, 334 insertions(+) create mode 100644 docs/icosagon-hierarchy.svg diff --git a/docs/icosagon-hierarchy.svg b/docs/icosagon-hierarchy.svg new file mode 100644 index 0000000..ed26cdd --- /dev/null +++ b/docs/icosagon-hierarchy.svg @@ -0,0 +1,334 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + Data + RelationFamily + + + + + RelationType + + + Adjacency Matrix + + Edges Pos + + Edges Pos + + + + + + + + + + + + -- 2.26.2 From 6c52bc1f71a292baf22a94278336c1447452a618 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 11 Jun 2020 14:17:50 +0200 Subject: [PATCH 081/227] Fix regressions in test_convlayer. --- src/icosagon/convlayer.py | 11 ++++--- src/icosagon/data.py | 21 ++++++++++++- tests/icosagon/test_convlayer.py | 52 +++++++++++++++++++++----------- 3 files changed, 62 insertions(+), 22 deletions(-) diff --git a/src/icosagon/convlayer.py b/src/icosagon/convlayer.py index 3fe1313..465f2a5 100644 --- a/src/icosagon/convlayer.py +++ b/src/icosagon/convlayer.py @@ -50,10 +50,8 @@ class DecagonLayer(torch.nn.Module): self.next_layer_repr = None self.build() - def build(self): - self.next_layer_repr = [ [] for _ in range(len(self.data.node_types)) ] - - for (node_type_row, node_type_column), rels in self.data.relation_types.items(): + def build_family(self, fam): + for (node_type_row, node_type_column), rels in fam.relation_types.items(): if len(rels) == 0: continue @@ -69,6 +67,11 @@ class DecagonLayer(torch.nn.Module): self.next_layer_repr[node_type_row].append( Convolutions(node_type_column, convolutions)) + def build(self): + self.next_layer_repr = [ [] for _ in range(len(self.data.node_types)) ] + for fam in self.data.relation_families: + self.build_family(fam) + def __call__(self, prev_layer_repr): next_layer_repr = [ [] for _ in range(len(self.data.node_types)) ] n = len(self.data.node_types) diff --git a/src/icosagon/data.py b/src/icosagon/data.py index cd55b29..b04432e 100644 --- a/src/icosagon/data.py +++ b/src/icosagon/data.py @@ -16,6 +16,24 @@ from .decode import DEDICOMDecoder, \ BilinearDecoder +def _equal(x: torch.Tensor, y: torch.Tensor): + if x.is_sparse ^ y.is_sparse: + raise ValueError('Cannot mix sparse and dense tensors') + + if not x.is_sparse: + return (x == y) + + x = x.coalesce() + indices_x = list(map(tuple, x.indices().transpose(0, 1))) + order_x = sorted(range(len(indices_x)), key=lambda idx: indices_x[idx]) + + y = y.coalesce() + indices_y = list(map(tuple, y.indices().transpose(0, 1))) + order_y = sorted(range(len(indices_y)), key=lambda idx: indices_y[idx]) + + return (x.values()[order_x] == y.values()[order_y]) + + @dataclass class NodeType(object): name: str @@ -97,7 +115,8 @@ class RelationFamily(object): raise ValueError('Cannot use a custom adjacency_matrix_backward in a symmetric relation family') if self.is_symmetric and node_type_row == node_type_column and \ - not torch.all(adjacency_matrix == adjacency_matrix.transpose(0, 1)): + not torch.all(_equal(adjacency_matrix, + adjacency_matrix.transpose(0, 1))): raise ValueError('Relation family is symmetric but adjacency_matrix is assymetric') two_way = bool(two_way) diff --git a/tests/icosagon/test_convlayer.py b/tests/icosagon/test_convlayer.py index 8e713c2..cc0458f 100644 --- a/tests/icosagon/test_convlayer.py +++ b/tests/icosagon/test_convlayer.py @@ -10,20 +10,36 @@ from decagon_pytorch.convolve import MultiDGCA import decagon_pytorch.convolve +def _make_symmetric(x: torch.Tensor): + x = (x + x.transpose(0, 1)) / 2 + return x + + +def _symmetric_random(n_rows, n_columns): + return _make_symmetric(torch.rand((n_rows, n_columns), + dtype=torch.float32).round()) + + def _some_data_with_interactions(): d = Data() d.add_node_type('Gene', 1000) d.add_node_type('Drug', 100) - d.add_relation_type('Target', 1, 0, + + fam = d.add_relation_family('Drug-Gene', 1, 0, True) + fam.add_relation_type('Target', 1, 0, torch.rand((100, 1000), dtype=torch.float32).round()) - d.add_relation_type('Interaction', 0, 0, - torch.rand((1000, 1000), dtype=torch.float32).round()) - d.add_relation_type('Side Effect: Nausea', 1, 1, - torch.rand((100, 100), dtype=torch.float32).round()) - d.add_relation_type('Side Effect: Infertility', 1, 1, - torch.rand((100, 100), dtype=torch.float32).round()) - d.add_relation_type('Side Effect: Death', 1, 1, - torch.rand((100, 100), dtype=torch.float32).round()) + + fam = d.add_relation_family('Gene-Gene', 0, 0, True) + fam.add_relation_type('Interaction', 0, 0, + _symmetric_random(1000, 1000)) + + fam = d.add_relation_family('Drug-Drug', 1, 1, True) + fam.add_relation_type('Side Effect: Nausea', 1, 1, + _symmetric_random(100, 100)) + fam.add_relation_type('Side Effect: Infertility', 1, 1, + _symmetric_random(100, 100)) + fam.add_relation_type('Side Effect: Death', 1, 1, + _symmetric_random(100, 100)) return d @@ -80,13 +96,14 @@ def test_decagon_layer_04(): d = Data() d.add_node_type('Dummy', 100) - d.add_relation_type('Dummy Relation', 0, 0, - torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, True) + fam.add_relation_type('Dummy Relation', 0, 0, + _symmetric_random(100, 100).to_sparse()) in_layer = OneHotInputLayer(d) multi_dgca = MultiDGCA([10], 32, - [r.adjacency_matrix for r in d.relation_types[0, 0]], + [r.adjacency_matrix for r in fam.relation_types[0, 0]], keep_prob=1., activation=lambda x: x) d_layer = DecagonLayer(in_layer.output_dim, 32, d, @@ -121,15 +138,16 @@ def test_decagon_layer_05(): d = Data() d.add_node_type('Dummy', 100) - d.add_relation_type('Dummy Relation 1', 0, 0, - torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) - d.add_relation_type('Dummy Relation 2', 0, 0, - torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, True) + fam.add_relation_type('Dummy Relation 1', 0, 0, + _symmetric_random(100, 100).to_sparse()) + fam.add_relation_type('Dummy Relation 2', 0, 0, + _symmetric_random(100, 100).to_sparse()) in_layer = OneHotInputLayer(d) multi_dgca = MultiDGCA([100, 100], 32, - [r.adjacency_matrix for r in d.relation_types[0, 0]], + [r.adjacency_matrix for r in fam.relation_types[0, 0]], keep_prob=1., activation=lambda x: x) d_layer = DecagonLayer(in_layer.output_dim, output_dim=32, data=d, -- 2.26.2 From 915c5684900a4dc1d9c1c7afa2a4690d8bf2a4c2 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 11 Jun 2020 14:23:16 +0200 Subject: [PATCH 082/227] Small fix. --- src/icosagon/data.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/icosagon/data.py b/src/icosagon/data.py index b04432e..0690d29 100644 --- a/src/icosagon/data.py +++ b/src/icosagon/data.py @@ -31,6 +31,9 @@ def _equal(x: torch.Tensor, y: torch.Tensor): indices_y = list(map(tuple, y.indices().transpose(0, 1))) order_y = sorted(range(len(indices_y)), key=lambda idx: indices_y[idx]) + if not indices_x == indices_y: + return torch.tensor(0, dtype=torch.uint8) + return (x.values()[order_x] == y.values()[order_y]) -- 2.26.2 From 7eda6bdfb90ef976a7d5891e82ee9e15b707ae7e Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 11 Jun 2020 18:28:22 +0200 Subject: [PATCH 083/227] Fix regressions in trainprep. --- src/icosagon/data.py | 47 ++++++++++++++++-------------- src/icosagon/trainprep.py | 50 ++++++++++++++++++++------------ tests/icosagon/test_declayer.py | 3 +- tests/icosagon/test_trainprep.py | 2 +- 4 files changed, 59 insertions(+), 43 deletions(-) diff --git a/src/icosagon/data.py b/src/icosagon/data.py index 0690d29..6fc91cc 100644 --- a/src/icosagon/data.py +++ b/src/icosagon/data.py @@ -44,38 +44,41 @@ class NodeType(object): @dataclass -class RelationType(object): +class RelationTypeBase(object): name: str node_type_row: int node_type_column: int adjacency_matrix: torch.Tensor two_way: bool + + +@dataclass +class RelationType(RelationTypeBase): hints: Dict[str, Any] = field(default_factory=dict) -class RelationFamily(object): - def __init__(self, - data: 'Data', - name: str, - node_type_row: int, - node_type_column: int, - is_symmetric: bool, - decoder_class: Type) -> None: +@dataclass +class RelationFamilyBase(object): + data: 'Data' + name: str + node_type_row: int + node_type_column: int + is_symmetric: bool + decoder_class: Type - if not is_symmetric and \ - decoder_class != DEDICOMDecoder and \ - decoder_class != BilinearDecoder: - raise TypeError('Family is assymetric but the specified decoder_class supports symmetric relations only') - self.data = data - self.name = name - self.node_type_row = node_type_row - self.node_type_column = node_type_column - self.is_symmetric = is_symmetric - self.decoder_class = decoder_class +@dataclass +class RelationFamily(RelationFamilyBase): + relation_types: Dict[Tuple[int, int], List[RelationType]] = None + + def __post_init__(self) -> None: + if not self.is_symmetric and \ + self.decoder_class != DEDICOMDecoder and \ + self.decoder_class != BilinearDecoder: + raise TypeError('Family is assymetric but the specified decoder_class supports symmetric relations only') - self.relation_types = { (node_type_row, node_type_column): [], - (node_type_column, node_type_row): [] } + self.relation_types = { (self.node_type_row, self.node_type_column): [], + (self.node_type_column, self.node_type_row): [] } def add_relation_type(self, name: str, node_type_row: int, node_type_column: int, adjacency_matrix: torch.Tensor, adjacency_matrix_backward: torch.Tensor = None, @@ -166,7 +169,7 @@ class RelationFamily(object): class Data(object): node_types: List[NodeType] - relation_types: Dict[Tuple[int, int], List[RelationType]] + relation_families: List[RelationFamily] def __init__(self) -> None: self.node_types = [] diff --git a/src/icosagon/trainprep.py b/src/icosagon/trainprep.py index a505d9b..d1dcdb3 100644 --- a/src/icosagon/trainprep.py +++ b/src/icosagon/trainprep.py @@ -6,13 +6,17 @@ from .sampling import fixed_unigram_candidate_sampler import torch -from dataclasses import dataclass +from dataclasses import dataclass, \ + field from typing import Any, \ List, \ Tuple, \ Dict from .data import NodeType, \ RelationType, \ + RelationTypeBase, \ + RelationFamily, \ + RelationFamilyBase, \ Data from collections import defaultdict from .normalize import norm_adj_mat_one_node_type, \ @@ -28,25 +32,20 @@ class TrainValTest(object): @dataclass -class PreparedEdges(object): - positive: TrainValTest - negative: TrainValTest +class PreparedRelationType(RelationTypeBase): + edges_pos: TrainValTest + edges_neg: TrainValTest @dataclass -class PreparedRelationType(object): - name: str - node_type_row: int - node_type_column: int - adjacency_matrix: torch.Tensor - edges_pos: TrainValTest - edges_neg: TrainValTest +class PreparedRelationFamily(RelationFamilyBase): + relation_types: Dict[Tuple[int, int], List[PreparedRelationType]] @dataclass class PreparedData(object): node_types: List[NodeType] - relation_types: Dict[Tuple[int, int], List[PreparedRelationType]] + relation_families: List[PreparedRelationFamily] def train_val_test_split_edges(edges: torch.Tensor, @@ -130,16 +129,29 @@ def prepare_relation_type(r: RelationType, adj_mat_train = norm_adj_mat_two_node_types(adj_mat_train) return PreparedRelationType(r.name, r.node_type_row, r.node_type_column, - adj_mat_train, edges_pos, edges_neg) + adj_mat_train, r.two_way, edges_pos, edges_neg) -def prepare_training(data: Data, ratios: TrainValTest) -> PreparedData: - if not isinstance(data, Data): - raise ValueError('data must be of class Data') +def prepare_relation_family(fam: RelationFamily) -> PreparedRelationFamily: + relation_types = { (fam.node_type_row, fam.node_type_column): [], + (fam.node_type_column, fam.node_type_row): [] } - relation_types = defaultdict(list) - for (node_type_row, node_type_column), rels in data.relation_types.items(): + for (node_type_row, node_type_column), rels in fam.relation_types.items(): for r in rels: relation_types[node_type_row, node_type_column].append( prepare_relation_type(r, ratios)) - return PreparedData(data.node_types, relation_types) + + return PreparedRelationFamily(fam.data, fam.name, + fam.node_type_row, fam.node_type_column, + fam.is_symmetric, fam.decoder_class, + relation_types) + + +def prepare_training(data: Data, ratios: TrainValTest) -> PreparedData: + if not isinstance(data, Data): + raise ValueError('data must be of class Data') + + relation_families = [ prepare_relation_family(fam) \ + for fam in data.relation_families ] + + return PreparedData(data.node_types, relation_families) diff --git a/tests/icosagon/test_declayer.py b/tests/icosagon/test_declayer.py index 3536387..7da2ad1 100644 --- a/tests/icosagon/test_declayer.py +++ b/tests/icosagon/test_declayer.py @@ -17,7 +17,8 @@ import torch def test_decode_layer_01(): d = Data() d.add_node_type('Dummy', 100) - d.add_relation_type('Dummy Relation 1', 0, 0, + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Relation 1', 0, 0, torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) in_layer = OneHotInputLayer(d) diff --git a/tests/icosagon/test_trainprep.py b/tests/icosagon/test_trainprep.py index 967bb1e..712d8c5 100644 --- a/tests/icosagon/test_trainprep.py +++ b/tests/icosagon/test_trainprep.py @@ -105,7 +105,7 @@ def test_prepare_adj_mat_02(): def test_prepare_relation_type_01(): adj_mat = (torch.rand((10, 10)) > .5) - r = RelationType('Test', 0, 0, adj_mat) + r = RelationType('Test', 0, 0, adj_mat, True) ratios = TrainValTest(.8, .1, .1) _ = prepare_relation_type(r, ratios) -- 2.26.2 From a72b6b09bcf5765847dd956e1d1b4cf7486fd585 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 11 Jun 2020 18:37:32 +0200 Subject: [PATCH 084/227] Fix regressions in test_input. --- tests/icosagon/test_input.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/tests/icosagon/test_input.py b/tests/icosagon/test_input.py index 1e9cfbf..1ac2e3d 100644 --- a/tests/icosagon/test_input.py +++ b/tests/icosagon/test_input.py @@ -9,28 +9,17 @@ def _some_data(): d = Data() d.add_node_type('Gene', 1000) d.add_node_type('Drug', 100) - d.add_relation_type('Target', 1, 0, torch.rand(100, 1000)) - d.add_relation_type('Interaction', 0, 0, torch.rand(1000, 1000)) - d.add_relation_type('Side Effect: Nausea', 1, 1, torch.rand(100, 100)) - d.add_relation_type('Side Effect: Infertility', 1, 1, torch.rand(100, 100)) - d.add_relation_type('Side Effect: Death', 1, 1, torch.rand(100, 100)) - return d + fam = d.add_relation_family('Drug-Gene', 1, 0, False) + fam.add_relation_type('Target', 1, 0, torch.rand(100, 1000)) -def _some_data_with_interactions(): - d = Data() - d.add_node_type('Gene', 1000) - d.add_node_type('Drug', 100) - d.add_relation_type('Target', 1, 0, - torch.rand((100, 1000), dtype=torch.float32).round()) - d.add_relation_type('Interaction', 0, 0, - torch.rand((1000, 1000), dtype=torch.float32).round()) - d.add_relation_type('Side Effect: Nausea', 1, 1, - torch.rand((100, 100), dtype=torch.float32).round()) - d.add_relation_type('Side Effect: Infertility', 1, 1, - torch.rand((100, 100), dtype=torch.float32).round()) - d.add_relation_type('Side Effect: Death', 1, 1, - torch.rand((100, 100), dtype=torch.float32).round()) + fam = d.add_relation_family('Gene-Gene', 0, 0, False) + fam.add_relation_type('Interaction', 0, 0, torch.rand(1000, 1000)) + + fam = d.add_relation_family('Drug-Drug', 1, 1, False) + fam.add_relation_type('Side Effect: Nausea', 1, 1, torch.rand(100, 100)) + fam.add_relation_type('Side Effect: Infertility', 1, 1, torch.rand(100, 100)) + fam.add_relation_type('Side Effect: Death', 1, 1, torch.rand(100, 100)) return d -- 2.26.2 From 832f620a7859c7d52bdbbb5ae5479dcb733fa68d Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 11 Jun 2020 22:48:22 +0200 Subject: [PATCH 085/227] Store relation types in a list instead of a dictionary and related fixes. --- src/icosagon/convlayer.py | 48 ++++++++++++++----- src/icosagon/data.py | 81 +++++++++++++++++--------------- src/icosagon/declayer.py | 20 ++++---- src/icosagon/trainprep.py | 79 ++++++++++++++++++++++++------- tests/icosagon/test_convlayer.py | 4 +- tests/icosagon/test_trainprep.py | 2 +- 6 files changed, 154 insertions(+), 80 deletions(-) diff --git a/src/icosagon/convlayer.py b/src/icosagon/convlayer.py index 465f2a5..cdaced6 100644 --- a/src/icosagon/convlayer.py +++ b/src/icosagon/convlayer.py @@ -50,22 +50,46 @@ class DecagonLayer(torch.nn.Module): self.next_layer_repr = None self.build() - def build_family(self, fam): - for (node_type_row, node_type_column), rels in fam.relation_types.items(): - if len(rels) == 0: - continue - - convolutions = [] + def build_fam_one_node_type(self, fam): + convolutions = [] + + for r in fam.relation_types: + conv = DropoutGraphConvActivation(self.input_dim[fam.node_type_column], + self.output_dim[fam.node_type_row], r.adjacency_matrix, + self.keep_prob, self.rel_activation) + convolutions.append(conv) + + self.next_layer_repr[fam.node_type_row].append( + Convolutions(fam.node_type_column, convolutions)) + + def build_fam_two_node_types(self, fam) -> None: + convolutions_row = [] + convolutions_column = [] + + for r in fam.relation_types: + if r.adjacency_matrix is not None: + conv = DropoutGraphConvActivation(self.input_dim[fam.node_type_column], + self.output_dim[fam.node_type_row], r.adjacency_matrix, + self.keep_prob, self.rel_activation) + convolutions_row.append(conv) - for r in rels: - conv = DropoutGraphConvActivation(self.input_dim[node_type_column], - self.output_dim[node_type_row], r.adjacency_matrix, + if r.adjacency_matrix_backward is not None: + conv = DropoutGraphConvActivation(self.input_dim[fam.node_type_row], + self.output_dim[fam.node_type_column], r.adjacency_matrix_backward, self.keep_prob, self.rel_activation) + convolutions_column.append(conv) + + self.next_layer_repr[fam.node_type_row].append( + Convolutions(fam.node_type_column, convolutions_row)) - convolutions.append(conv) + self.next_layer_repr[fam.node_type_column].append( + Convolutions(fam.node_type_row, convolutions_column)) - self.next_layer_repr[node_type_row].append( - Convolutions(node_type_column, convolutions)) + def build_family(self, fam) -> None: + if fam.node_type_row == fam.node_type_column: + self.build_fam_one_node_type(fam) + else: + self.build_fam_two_node_types(fam) def build(self): self.next_layer_repr = [ [] for _ in range(len(self.data.node_types)) ] diff --git a/src/icosagon/data.py b/src/icosagon/data.py index 6fc91cc..53bd769 100644 --- a/src/icosagon/data.py +++ b/src/icosagon/data.py @@ -49,12 +49,12 @@ class RelationTypeBase(object): node_type_row: int node_type_column: int adjacency_matrix: torch.Tensor - two_way: bool + adjacency_matrix_backward: torch.Tensor @dataclass class RelationType(RelationTypeBase): - hints: Dict[str, Any] = field(default_factory=dict) + pass @dataclass @@ -69,7 +69,7 @@ class RelationFamilyBase(object): @dataclass class RelationFamily(RelationFamilyBase): - relation_types: Dict[Tuple[int, int], List[RelationType]] = None + relation_types: List[RelationType] = None def __post_init__(self) -> None: if not self.is_symmetric and \ @@ -77,18 +77,18 @@ class RelationFamily(RelationFamilyBase): self.decoder_class != BilinearDecoder: raise TypeError('Family is assymetric but the specified decoder_class supports symmetric relations only') - self.relation_types = { (self.node_type_row, self.node_type_column): [], - (self.node_type_column, self.node_type_row): [] } + self.relation_types = [] - def add_relation_type(self, name: str, node_type_row: int, node_type_column: int, - adjacency_matrix: torch.Tensor, adjacency_matrix_backward: torch.Tensor = None, - two_way: bool = True) -> None: + def add_relation_type(self, + name: str, node_type_row: int, node_type_column: int, + adjacency_matrix: torch.Tensor, + adjacency_matrix_backward: torch.Tensor = None) -> None: name = str(name) node_type_row = int(node_type_row) node_type_column = int(node_type_column) - if (node_type_row, node_type_column) not in self.relation_types: + if (node_type_row, node_type_column) != (self.node_type_row, self.node_type_column): raise ValueError('Specified node_type_row/node_type_column tuple does not belong to this family') if node_type_row < 0 or node_type_row >= len(self.data.node_types): @@ -97,21 +97,33 @@ class RelationFamily(RelationFamilyBase): if node_type_column < 0 or node_type_column >= len(self.data.node_types): raise ValueError('node_type_column outside of the valid range of node types') - if not isinstance(adjacency_matrix, torch.Tensor): + if adjacency_matrix is None and adjacency_matrix_backward is None: + raise ValueError('adjacency_matrix and adjacency_matrix_backward cannot both be None') + + if adjacency_matrix is not None and \ + not isinstance(adjacency_matrix, torch.Tensor): raise ValueError('adjacency_matrix must be a torch.Tensor') + # if isinstance(adjacency_matrix_backward, str) and \ + # adjacency_matrix_backward == 'symmetric': + # if self.is_symmetric: + # adjacency_matrix_backward = None + # else: + # adjacency_matrix_backward = adjacency_matrix.transpose(0, 1) + if adjacency_matrix_backward is not None \ and not isinstance(adjacency_matrix_backward, torch.Tensor): raise ValueError('adjacency_matrix_backward must be a torch.Tensor') - if adjacency_matrix.shape != (self.data.node_types[node_type_row].count, - self.data.node_types[node_type_column].count): + if adjacency_matrix is not None and \ + adjacency_matrix.shape != (self.data.node_types[node_type_row].count, + self.data.node_types[node_type_column].count): raise ValueError('adjacency_matrix shape must be (num_row_nodes, num_column_nodes)') if adjacency_matrix_backward is not None and \ adjacency_matrix_backward.shape != (self.data.node_types[node_type_column].count, self.data.node_types[node_type_row].count): - raise ValueError('adjacency_matrix shape must be (num_column_nodes, num_row_nodes)') + raise ValueError('adjacency_matrix_backward shape must be (num_column_nodes, num_row_nodes)') if node_type_row == node_type_column and \ adjacency_matrix_backward is not None: @@ -125,19 +137,12 @@ class RelationFamily(RelationFamilyBase): adjacency_matrix.transpose(0, 1))): raise ValueError('Relation family is symmetric but adjacency_matrix is assymetric') - two_way = bool(two_way) - - if node_type_row != node_type_column and two_way: - print('%d != %d' % (node_type_row, node_type_column)) - if adjacency_matrix_backward is None: - adjacency_matrix_backward = adjacency_matrix.transpose(0, 1) - self.relation_types[node_type_column, node_type_row].append( - RelationType(name, node_type_column, node_type_row, - adjacency_matrix_backward, two_way, { 'display': False })) + if self.is_symmetric and node_type_row != node_type_column: + adjacency_matrix_backward = adjacency_matrix.transpose(0, 1) - self.relation_types[node_type_row, node_type_column].append( - RelationType(name, node_type_row, node_type_column, - adjacency_matrix, two_way)) + self.relation_types.append(RelationType(name, + node_type_row, node_type_column, + adjacency_matrix, adjacency_matrix_backward)) def node_name(self, index): return self.data.node_types[index].name @@ -145,24 +150,26 @@ class RelationFamily(RelationFamilyBase): def __repr__(self): s = 'Relation family %s' % self.name - for (node_type_row, node_type_column), rels in self.relation_types.items(): - for r in rels: - if 'display' in r.hints and not r.hints['display']: - continue - s += '\n - %s%s' % (r.name, ' (two-way)' if r.two_way else '%s <- %s' % \ - (self.node_name(node_type_row), self.node_name(node_type_column))) + for r in self.relation_types: + s += '\n - %s%s' % (r.name, ' (two-way)' \ + if (r.adjacency_matrix is not None \ + and r.adjacency_matrix_backward is not None) \ + or self.is_symmetric \ + else '%s <- %s' % (self.node_name(self.node_type_row), + self.node_name(self.node_type_column))) return s def repr_indented(self): s = ' - %s' % self.name - for (node_type_row, node_type_column), rels in self.relation_types.items(): - for r in rels: - if 'display' in r.hints and not r.hints['display']: - continue - s += '\n - %s%s' % (r.name, ' (two-way)' if r.two_way else '%s <- %s' % \ - (self.node_name(node_type_row), self.node_name(node_type_column))) + for r in self.relation_types: + s += '\n - %s%s' % (r.name, ' (two-way)' \ + if (r.adjacency_matrix is not None \ + and r.adjacency_matrix_backward is not None) \ + or self.is_symmetric \ + else '%s <- %s' % (self.node_name(self.node_type_row), + self.node_name(self.node_type_column))) return s diff --git a/src/icosagon/declayer.py b/src/icosagon/declayer.py index 9047025..7840ab5 100644 --- a/src/icosagon/declayer.py +++ b/src/icosagon/declayer.py @@ -22,7 +22,6 @@ class DecodeLayer(torch.nn.Module): input_dim: List[int], data: PreparedData, keep_prob: float = 1., - decoder_class: Union[Type, Dict[Tuple[int, int], Type]] = DEDICOMDecoder, activation: Callable[[torch.Tensor], torch.Tensor] = torch.sigmoid, **kwargs) -> None: @@ -37,26 +36,25 @@ class DecodeLayer(torch.nn.Module): if not isinstance(data, PreparedData): raise TypeError('data must be an instance of PreparedData') - if not isinstance(decoder_class, type) and \ - not isinstance(decoder_class, dict): - raise TypeError('decoder_class must be a Type or a Dict') - - if not isinstance(decoder_class, dict): - decoder_class = { k: decoder_class \ - for k in data.relation_types.keys() } - self.input_dim = input_dim self.output_dim = 1 self.data = data self.keep_prob = keep_prob - self.decoder_class = decoder_class self.activation = activation self.decoders = None self.build() def build(self) -> None: - self.decoders = {} + self.decoders = [] + + for fam in self.data.relation_families: + for (node_type_row, node_type_column), rels in fam.relation_types.items(): + for r in rels: + pass + + dec = fam.decoder_class() + self.decoders.append(dec) for (node_type_row, node_type_column), rels in self.data.relation_types.items(): if len(rels) == 0: diff --git a/src/icosagon/trainprep.py b/src/icosagon/trainprep.py index d1dcdb3..5c7843d 100644 --- a/src/icosagon/trainprep.py +++ b/src/icosagon/trainprep.py @@ -39,7 +39,7 @@ class PreparedRelationType(RelationTypeBase): @dataclass class PreparedRelationFamily(RelationFamilyBase): - relation_types: Dict[Tuple[int, int], List[PreparedRelationType]] + relation_types: List[PreparedRelationType] @dataclass @@ -110,36 +110,81 @@ def prepare_adj_mat(adj_mat: torch.Tensor, return adj_mat_train, edges_pos, edges_neg -def prepare_relation_type(r: RelationType, +def prep_rel_one_node_type(r: RelationType, ratios: TrainValTest) -> PreparedRelationType: + adj_mat = r.adjacency_matrix + adj_mat_train, edges_pos, edges_neg = prepare_adj_mat(adj_mat, ratios) + + print('adj_mat_train:', adj_mat_train) + adj_mat_train = norm_adj_mat_one_node_type(adj_mat_train) + + return PreparedRelationType(r.name, r.node_type_row, r.node_type_column, + adj_mat_train, None, edges_pos, edges_neg) + + +def prep_rel_two_node_types_sym(r: RelationType, + ratios: TrainValTest) -> PreparedRelationType: + + adj_mat = r.adjacency_matrix + adj_mat_train, edges_pos, edges_neg = prepare_adj_mat(adj_mat, ratios) + + return PreparedRelationType(r.name, r.node_type_row, + r.node_type_column, + norm_adj_mat_two_node_types(adj_mat_train), + norm_adj_mat_two_node_types(adj_mat_train.transpose(0, 1)), + edges_pos, edges_neg) + + +def prep_rel_two_node_types_asym(r: RelationType, + ratios: TrainValTest) -> PreparedRelationType: + + if r.adjacency_matrix is not None: + adj_mat_train, edges_pos, edges_neg =\ + prepare_adj_mat(r.adjacency_matrix, ratios) + else: + adj_mat_train, edges_pos, edges_neg = \ + None, torch.zeros((0, 2)), torch.zeros((0, 2)) + + if r.adjacency_matrix_backward is not None: + adj_mat_back_train, edges_back_pos, edges_back_neg = \ + prepare_adj_mat(r.adjacency_matrix_backward, ratios) + else: + adj_mat_back_train, edges_back_pos, edges_back_neg = \ + None, torch.zeros((0, 2)), torch.zeros((0, 2)) + + edges_pos = torch.cat((edges_pos, edges_back_pos), dim=0) + edges_neg = torch.cat((edges_neg, edges_back_neg), dim=0) + + return PreparedRelationType(r.name, r.node_type_row, + r.node_type_column, + norm_adj_mat_two_node_types(adj_mat_train), + norm_adj_mat_two_node_types(adj_mat_back_train), + edges_pos, edges_neg) + + +def prepare_relation_type(r: RelationType, + ratios: TrainValTest, is_symmetric: bool) -> PreparedRelationType: + if not isinstance(r, RelationType): raise ValueError('r must be a RelationType') if not isinstance(ratios, TrainValTest): raise ValueError('ratios must be a TrainValTest') - adj_mat = r.adjacency_matrix - adj_mat_train, edges_pos, edges_neg = prepare_adj_mat(adj_mat, ratios) - - print('adj_mat_train:', adj_mat_train) if r.node_type_row == r.node_type_column: - adj_mat_train = norm_adj_mat_one_node_type(adj_mat_train) + return prep_rel_one_node_type(r, ratios) + elif is_symmetric: + return prep_rel_two_node_types_sym(r, ratios) else: - adj_mat_train = norm_adj_mat_two_node_types(adj_mat_train) - - return PreparedRelationType(r.name, r.node_type_row, r.node_type_column, - adj_mat_train, r.two_way, edges_pos, edges_neg) + return prep_rel_two_node_types_asym(r, ratios) def prepare_relation_family(fam: RelationFamily) -> PreparedRelationFamily: - relation_types = { (fam.node_type_row, fam.node_type_column): [], - (fam.node_type_column, fam.node_type_row): [] } + relation_types = [] - for (node_type_row, node_type_column), rels in fam.relation_types.items(): - for r in rels: - relation_types[node_type_row, node_type_column].append( - prepare_relation_type(r, ratios)) + for r in fam.relation_types: + relation_types.append(prepare_relation_type(r, ratios, fam.is_symmetric)) return PreparedRelationFamily(fam.data, fam.name, fam.node_type_row, fam.node_type_column, diff --git a/tests/icosagon/test_convlayer.py b/tests/icosagon/test_convlayer.py index cc0458f..11e8ffc 100644 --- a/tests/icosagon/test_convlayer.py +++ b/tests/icosagon/test_convlayer.py @@ -103,7 +103,7 @@ def test_decagon_layer_04(): in_layer = OneHotInputLayer(d) multi_dgca = MultiDGCA([10], 32, - [r.adjacency_matrix for r in fam.relation_types[0, 0]], + [r.adjacency_matrix for r in fam.relation_types], keep_prob=1., activation=lambda x: x) d_layer = DecagonLayer(in_layer.output_dim, 32, d, @@ -147,7 +147,7 @@ def test_decagon_layer_05(): in_layer = OneHotInputLayer(d) multi_dgca = MultiDGCA([100, 100], 32, - [r.adjacency_matrix for r in fam.relation_types[0, 0]], + [r.adjacency_matrix for r in fam.relation_types], keep_prob=1., activation=lambda x: x) d_layer = DecagonLayer(in_layer.output_dim, output_dim=32, data=d, diff --git a/tests/icosagon/test_trainprep.py b/tests/icosagon/test_trainprep.py index 712d8c5..b3a2d1f 100644 --- a/tests/icosagon/test_trainprep.py +++ b/tests/icosagon/test_trainprep.py @@ -107,7 +107,7 @@ def test_prepare_relation_type_01(): adj_mat = (torch.rand((10, 10)) > .5) r = RelationType('Test', 0, 0, adj_mat, True) ratios = TrainValTest(.8, .1, .1) - _ = prepare_relation_type(r, ratios) + _ = prepare_relation_type(r, ratios, False) -- 2.26.2 From e8122c3321563409d87c8496a5e86b0bf7916fea Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 12 Jun 2020 15:16:50 +0200 Subject: [PATCH 086/227] Get test_decode_layer_01 to pass. --- src/icosagon/declayer.py | 90 ++++++++++++++++++++------------- src/icosagon/trainprep.py | 30 +++++++---- tests/icosagon/test_declayer.py | 32 +++++++++--- 3 files changed, 99 insertions(+), 53 deletions(-) diff --git a/src/icosagon/declayer.py b/src/icosagon/declayer.py index 7840ab5..db78f11 100644 --- a/src/icosagon/declayer.py +++ b/src/icosagon/declayer.py @@ -15,6 +15,25 @@ from typing import Type, \ Dict, \ Tuple from .decode import DEDICOMDecoder +from dataclasses import dataclass + + +@dataclass +class RelationPredictions(object): + edges_pos: TrainValTest + edges_neg: TrainValTest + edges_back_pos: TrainValTest + edges_back_neg: TrainValTest + + +@dataclass +class RelationFamilyPredictions(object): + relation_types: List[RelationPredictions] + + +@dataclass +class Predictions(object): + relation_families: List[RelationFamilyPredictions] class DecodeLayer(torch.nn.Module): @@ -30,13 +49,16 @@ class DecodeLayer(torch.nn.Module): if not isinstance(input_dim, list): raise TypeError('input_dim must be a List') + if len(input_dim) != len(data.node_types): + raise ValueError('input_dim must have length equal to num_node_types') + if not all([ a == input_dim[0] for a in input_dim ]): raise ValueError('All elements of input_dim must have the same value') if not isinstance(data, PreparedData): raise TypeError('data must be an instance of PreparedData') - self.input_dim = input_dim + self.input_dim = input_dim[0] self.output_dim = 1 self.data = data self.keep_prob = keep_prob @@ -47,42 +69,38 @@ class DecodeLayer(torch.nn.Module): def build(self) -> None: self.decoders = [] - for fam in self.data.relation_families: - for (node_type_row, node_type_column), rels in fam.relation_types.items(): - for r in rels: - pass - - dec = fam.decoder_class() + dec = fam.decoder_class(self.input_dim, len(fam.relation_types), + self.keep_prob, self.activation) self.decoders.append(dec) - for (node_type_row, node_type_column), rels in self.data.relation_types.items(): - if len(rels) == 0: - continue - - if isinstance(self.decoder_class, dict): - if (node_type_row, node_type_column) in self.decoder_class: - decoder_class = self.decoder_class[node_type_row, node_type_column] - elif (node_type_column, node_type_row) in self.decoder_class: - decoder_class = self.decoder_class[node_type_column, node_type_row] - else: - raise KeyError('Decoder not specified for edge type: %s -- %s' % ( - self.data.node_types[node_type_row].name, - self.data.node_types[node_type_column].name)) - else: - decoder_class = self.decoder_class - - self.decoders[node_type_row, node_type_column] = \ - decoder_class(self.input_dim[node_type_row], - num_relation_types = len(rels), - keep_prob = self.keep_prob, - activation = self.activation) - - def forward(self, last_layer_repr: List[torch.Tensor]) -> Dict[Tuple[int, int], List[torch.Tensor]]: - res = {} - for (node_type_row, node_type_column), dec in self.decoders.items(): - inputs_row = last_layer_repr[node_type_row] - inputs_column = last_layer_repr[node_type_column] - pred_adj_matrices = [ dec(inputs_row, inputs_column, k) for k in range(dec.num_relation_types) ] - res[node_type_row, node_type_column] = pred_adj_matrices + def _get_tvt(self, r, edge_list_attr_names, row, column, k, last_layer_repr, dec): + pred = [] + for p in edge_list_attr_names: + tvt = [] + for t in ['train', 'val', 'test']: + # print('r:', r) + edges = getattr(getattr(r, p), t) + inputs_row = last_layer_repr[row][edges[:, 0]] + inputs_column = last_layer_repr[column][edges[:, 1]] + tvt.append(dec(inputs_row, inputs_column, k)) + tvt = TrainValTest(*tvt) + pred.append(tvt) + return pred + + def forward(self, last_layer_repr: List[torch.Tensor]) -> List[List[torch.Tensor]]: + res = [] + for i, fam in enumerate(self.data.relation_families): + fam_pred = [] + for k, r in enumerate(fam.relation_types): + pred = [] + pred += self._get_tvt(r, ['edges_pos', 'edges_neg'], + r.node_type_row, r.node_type_column, k, last_layer_repr, self.decoders[i]) + pred += self._get_tvt(r, ['edges_back_pos', 'edges_back_neg'], + r.node_type_column, r.node_type_row, k, last_layer_repr, self.decoders[i]) + pred = RelationPredictions(*pred) + fam_pred.append(pred) + fam_pred = RelationFamilyPredictions(fam_pred) + res.append(fam_pred) + res = Predictions(res) return res diff --git a/src/icosagon/trainprep.py b/src/icosagon/trainprep.py index 5c7843d..3457791 100644 --- a/src/icosagon/trainprep.py +++ b/src/icosagon/trainprep.py @@ -35,6 +35,8 @@ class TrainValTest(object): class PreparedRelationType(RelationTypeBase): edges_pos: TrainValTest edges_neg: TrainValTest + edges_back_pos: TrainValTest + edges_back_neg: TrainValTest @dataclass @@ -48,6 +50,10 @@ class PreparedData(object): relation_families: List[PreparedRelationFamily] +def _empty_edge_list_tvt() -> TrainValTest: + return TrainValTest(*[ torch.zeros((0, 2), dtype=torch.long) for _ in range(3) ]) + + def train_val_test_split_edges(edges: torch.Tensor, ratios: TrainValTest) -> TrainValTest: @@ -115,12 +121,15 @@ def prep_rel_one_node_type(r: RelationType, adj_mat = r.adjacency_matrix adj_mat_train, edges_pos, edges_neg = prepare_adj_mat(adj_mat, ratios) + adj_mat_back_train, edges_back_pos, edges_back_neg = \ + None, _empty_edge_list_tvt(), _empty_edge_list_tvt() print('adj_mat_train:', adj_mat_train) adj_mat_train = norm_adj_mat_one_node_type(adj_mat_train) return PreparedRelationType(r.name, r.node_type_row, r.node_type_column, - adj_mat_train, None, edges_pos, edges_neg) + adj_mat_train, adj_mat_back_train, edges_pos, edges_neg, + edges_back_pos, edges_back_neg) def prep_rel_two_node_types_sym(r: RelationType, @@ -128,12 +137,14 @@ def prep_rel_two_node_types_sym(r: RelationType, adj_mat = r.adjacency_matrix adj_mat_train, edges_pos, edges_neg = prepare_adj_mat(adj_mat, ratios) + edges_back_pos, edges_back_neg = \ + _empty_edge_list_tvt(), _empty_edge_list_tvt() return PreparedRelationType(r.name, r.node_type_row, r.node_type_column, norm_adj_mat_two_node_types(adj_mat_train), norm_adj_mat_two_node_types(adj_mat_train.transpose(0, 1)), - edges_pos, edges_neg) + edges_pos, edges_neg, edges_back_pos, edges_back_neg) def prep_rel_two_node_types_asym(r: RelationType, @@ -144,23 +155,20 @@ def prep_rel_two_node_types_asym(r: RelationType, prepare_adj_mat(r.adjacency_matrix, ratios) else: adj_mat_train, edges_pos, edges_neg = \ - None, torch.zeros((0, 2)), torch.zeros((0, 2)) + None, _empty_edge_list_tvt(), _empty_edge_list_tvt() if r.adjacency_matrix_backward is not None: adj_mat_back_train, edges_back_pos, edges_back_neg = \ prepare_adj_mat(r.adjacency_matrix_backward, ratios) else: adj_mat_back_train, edges_back_pos, edges_back_neg = \ - None, torch.zeros((0, 2)), torch.zeros((0, 2)) - - edges_pos = torch.cat((edges_pos, edges_back_pos), dim=0) - edges_neg = torch.cat((edges_neg, edges_back_neg), dim=0) + None, _empty_edge_list_tvt(), _empty_edge_list_tvt() return PreparedRelationType(r.name, r.node_type_row, r.node_type_column, norm_adj_mat_two_node_types(adj_mat_train), norm_adj_mat_two_node_types(adj_mat_back_train), - edges_pos, edges_neg) + edges_pos, edges_neg, edges_back_pos, edges_back_neg) def prepare_relation_type(r: RelationType, @@ -180,7 +188,9 @@ def prepare_relation_type(r: RelationType, return prep_rel_two_node_types_asym(r, ratios) -def prepare_relation_family(fam: RelationFamily) -> PreparedRelationFamily: +def prepare_relation_family(fam: RelationFamily, + ratios: TrainValTest) -> PreparedRelationFamily: + relation_types = [] for r in fam.relation_types: @@ -196,7 +206,7 @@ def prepare_training(data: Data, ratios: TrainValTest) -> PreparedData: if not isinstance(data, Data): raise ValueError('data must be of class Data') - relation_families = [ prepare_relation_family(fam) \ + relation_families = [ prepare_relation_family(fam, ratios) \ for fam in data.relation_families ] return PreparedData(data.node_types, relation_families) diff --git a/tests/icosagon/test_declayer.py b/tests/icosagon/test_declayer.py index 7da2ad1..bab9a25 100644 --- a/tests/icosagon/test_declayer.py +++ b/tests/icosagon/test_declayer.py @@ -6,7 +6,10 @@ from icosagon.input import OneHotInputLayer from icosagon.convlayer import DecagonLayer -from icosagon.declayer import DecodeLayer +from icosagon.declayer import DecodeLayer, \ + Predictions, \ + RelationFamilyPredictions, \ + RelationPredictions from icosagon.decode import DEDICOMDecoder from icosagon.data import Data from icosagon.trainprep import prepare_training, \ @@ -17,21 +20,36 @@ import torch def test_decode_layer_01(): d = Data() d.add_node_type('Dummy', 100) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) fam.add_relation_type('Dummy Relation 1', 0, 0, torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) in_layer = OneHotInputLayer(d) d_layer = DecagonLayer(in_layer.output_dim, 32, d) seq = torch.nn.Sequential(in_layer, d_layer) last_layer_repr = seq(None) + dec = DecodeLayer(input_dim=d_layer.output_dim, data=prep_d, keep_prob=1., - decoder_class=DEDICOMDecoder, activation=lambda x: x) - pred_adj_matrices = dec(last_layer_repr) - assert isinstance(pred_adj_matrices, dict) - assert len(pred_adj_matrices) == 1 - assert isinstance(pred_adj_matrices[0, 0], list) - assert len(pred_adj_matrices[0, 0]) == 1 + activation=lambda x: x) + pred = dec(last_layer_repr) + + assert isinstance(pred, Predictions) + + assert isinstance(pred.relation_families, list) + assert len(pred.relation_families) == 1 + assert isinstance(pred.relation_families[0], RelationFamilyPredictions) + + assert isinstance(pred.relation_families[0].relation_types, list) + assert len(pred.relation_families[0].relation_types) == 1 + assert isinstance(pred.relation_families[0].relation_types[0], RelationPredictions) + + tmp = pred.relation_families[0].relation_types[0] + assert isinstance(tmp.edges_pos, TrainValTest) + assert isinstance(tmp.edges_neg, TrainValTest) + assert isinstance(tmp.edges_back_pos, TrainValTest) + assert isinstance(tmp.edges_back_neg, TrainValTest) def test_decode_layer_02(): -- 2.26.2 From 8767db601dcb19804143d98055c33580ac58c709 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 12 Jun 2020 16:14:03 +0200 Subject: [PATCH 087/227] Remove node_type_row/node_type_column as redundant in add_relation_type(). --- src/icosagon/data.py | 38 ++++++------- tests/icosagon/test_convlayer.py | 16 +++--- tests/icosagon/test_data.py | 97 ++++++++++++++++++++++++++++---- tests/icosagon/test_declayer.py | 2 +- tests/icosagon/test_decode.py | 1 - tests/icosagon/test_input.py | 10 ++-- 6 files changed, 115 insertions(+), 49 deletions(-) diff --git a/src/icosagon/data.py b/src/icosagon/data.py index 53bd769..0181b7d 100644 --- a/src/icosagon/data.py +++ b/src/icosagon/data.py @@ -80,22 +80,12 @@ class RelationFamily(RelationFamilyBase): self.relation_types = [] def add_relation_type(self, - name: str, node_type_row: int, node_type_column: int, - adjacency_matrix: torch.Tensor, + name: str, adjacency_matrix: torch.Tensor, adjacency_matrix_backward: torch.Tensor = None) -> None: name = str(name) - node_type_row = int(node_type_row) - node_type_column = int(node_type_column) - - if (node_type_row, node_type_column) != (self.node_type_row, self.node_type_column): - raise ValueError('Specified node_type_row/node_type_column tuple does not belong to this family') - - if node_type_row < 0 or node_type_row >= len(self.data.node_types): - raise ValueError('node_type_row outside of the valid range of node types') - - if node_type_column < 0 or node_type_column >= len(self.data.node_types): - raise ValueError('node_type_column outside of the valid range of node types') + node_type_row = self.node_type_row + node_type_column = self.node_type_column if adjacency_matrix is None and adjacency_matrix_backward is None: raise ValueError('adjacency_matrix and adjacency_matrix_backward cannot both be None') @@ -104,13 +94,6 @@ class RelationFamily(RelationFamilyBase): not isinstance(adjacency_matrix, torch.Tensor): raise ValueError('adjacency_matrix must be a torch.Tensor') - # if isinstance(adjacency_matrix_backward, str) and \ - # adjacency_matrix_backward == 'symmetric': - # if self.is_symmetric: - # adjacency_matrix_backward = None - # else: - # adjacency_matrix_backward = adjacency_matrix.transpose(0, 1) - if adjacency_matrix_backward is not None \ and not isinstance(adjacency_matrix_backward, torch.Tensor): raise ValueError('adjacency_matrix_backward must be a torch.Tensor') @@ -154,7 +137,7 @@ class RelationFamily(RelationFamilyBase): s += '\n - %s%s' % (r.name, ' (two-way)' \ if (r.adjacency_matrix is not None \ and r.adjacency_matrix_backward is not None) \ - or self.is_symmetric \ + or self.node_type_row == self.node_type_column \ else '%s <- %s' % (self.node_name(self.node_type_row), self.node_name(self.node_type_column))) @@ -167,7 +150,7 @@ class RelationFamily(RelationFamilyBase): s += '\n - %s%s' % (r.name, ' (two-way)' \ if (r.adjacency_matrix is not None \ and r.adjacency_matrix_backward is not None) \ - or self.is_symmetric \ + or self.node_type_row == self.node_type_column \ else '%s <- %s' % (self.node_name(self.node_type_row), self.node_name(self.node_type_column))) @@ -195,6 +178,17 @@ class Data(object): node_type_column: int, is_symmetric: bool, decoder_class: Type = DEDICOMDecoder): + name = str(name) + node_type_row = int(node_type_row) + node_type_column = int(node_type_column) + is_symmetric = bool(is_symmetric) + + if node_type_row < 0 or node_type_row >= len(self.node_types): + raise ValueError('node_type_row outside of the valid range of node types') + + if node_type_column < 0 or node_type_column >= len(self.node_types): + raise ValueError('node_type_column outside of the valid range of node types') + fam = RelationFamily(self, name, node_type_row, node_type_column, is_symmetric, decoder_class) self.relation_families.append(fam) diff --git a/tests/icosagon/test_convlayer.py b/tests/icosagon/test_convlayer.py index 11e8ffc..145ad95 100644 --- a/tests/icosagon/test_convlayer.py +++ b/tests/icosagon/test_convlayer.py @@ -26,19 +26,19 @@ def _some_data_with_interactions(): d.add_node_type('Drug', 100) fam = d.add_relation_family('Drug-Gene', 1, 0, True) - fam.add_relation_type('Target', 1, 0, + fam.add_relation_type('Target', torch.rand((100, 1000), dtype=torch.float32).round()) fam = d.add_relation_family('Gene-Gene', 0, 0, True) - fam.add_relation_type('Interaction', 0, 0, + fam.add_relation_type('Interaction', _symmetric_random(1000, 1000)) fam = d.add_relation_family('Drug-Drug', 1, 1, True) - fam.add_relation_type('Side Effect: Nausea', 1, 1, + fam.add_relation_type('Side Effect: Nausea', _symmetric_random(100, 100)) - fam.add_relation_type('Side Effect: Infertility', 1, 1, + fam.add_relation_type('Side Effect: Infertility', _symmetric_random(100, 100)) - fam.add_relation_type('Side Effect: Death', 1, 1, + fam.add_relation_type('Side Effect: Death', _symmetric_random(100, 100)) return d @@ -97,7 +97,7 @@ def test_decagon_layer_04(): d = Data() d.add_node_type('Dummy', 100) fam = d.add_relation_family('Dummy-Dummy', 0, 0, True) - fam.add_relation_type('Dummy Relation', 0, 0, + fam.add_relation_type('Dummy Relation', _symmetric_random(100, 100).to_sparse()) in_layer = OneHotInputLayer(d) @@ -139,9 +139,9 @@ def test_decagon_layer_05(): d = Data() d.add_node_type('Dummy', 100) fam = d.add_relation_family('Dummy-Dummy', 0, 0, True) - fam.add_relation_type('Dummy Relation 1', 0, 0, + fam.add_relation_type('Dummy Relation 1', _symmetric_random(100, 100).to_sparse()) - fam.add_relation_type('Dummy Relation 2', 0, 0, + fam.add_relation_type('Dummy Relation 2', _symmetric_random(100, 100).to_sparse()) in_layer = OneHotInputLayer(d) diff --git a/tests/icosagon/test_data.py b/tests/icosagon/test_data.py index d67c1c1..08ded89 100644 --- a/tests/icosagon/test_data.py +++ b/tests/icosagon/test_data.py @@ -4,11 +4,80 @@ # -from icosagon import Data +from icosagon.data import Data, \ + _equal, \ + RelationFamily +from icosagon.decode import DEDICOMDecoder import torch import pytest +def test_equal_01(): + x = torch.rand((10, 10)) + y = torch.rand((10, 10)).round().to_sparse() + + assert torch.all(_equal(x, x)) + assert torch.all(_equal(y, y)) + with pytest.raises(ValueError): + _equal(x, y) + + z = torch.rand((10, 10)).round().to_sparse() + assert not torch.all(_equal(y, z)) + + +def test_relation_family_01(): + d = Data() + d.add_node_type('Whatever', 10) + + fam = RelationFamily(d, 'Dummy-Dummy', 0, 0, True, DEDICOMDecoder) + + with pytest.raises(ValueError): + fam.add_relation_type('Dummy-Dummy', None, None) + + with pytest.raises(ValueError): + fam.add_relation_type('Dummy-Dummy', 'bad-value', None) + + with pytest.raises(ValueError): + fam.add_relation_type('Dummy-Dummy', None, 'bad-value') + + with pytest.raises(ValueError): + fam.add_relation_type('Dummy-Dummy', torch.rand((5, 5)), None) + + with pytest.raises(ValueError): + fam.add_relation_type('Dummy-Dummy', None, torch.rand((5, 5))) + + with pytest.raises(ValueError): + fam.add_relation_type('Dummy-Dummy', torch.rand((10, 10)), torch.rand((10, 10))) + + with pytest.raises(ValueError): + fam.add_relation_type('Dummy-Dummy', torch.rand((10, 10)), None) + + +def test_relation_family_02(): + d = Data() + d.add_node_type('A', 10) + d.add_node_type('B', 5) + + fam = RelationFamily(d, 'A-B', 0, 1, True, DEDICOMDecoder) + + with pytest.raises(ValueError): + fam.add_relation_type('A-B', torch.rand((10, 5)).round(), + torch.rand((5, 10)).round()) + + +def test_relation_family_03(): + d = Data() + d.add_node_type('A', 10) + d.add_node_type('B', 5) + + fam = RelationFamily(d, 'A-B', 0, 1, True, DEDICOMDecoder) + + fam.add_relation_type('A-B', torch.rand((10, 5)).round()) + + assert torch.all(fam.relation_types[0].adjacency_matrix.transpose(0, 1) == \ + fam.relation_types[0].adjacency_matrix_backward) + + def test_data_01(): d = Data() d.add_node_type('Gene', 1000) @@ -17,14 +86,18 @@ def test_data_01(): dummy_1 = torch.zeros((1000, 100)) dummy_2 = torch.zeros((100, 100)) dummy_3 = torch.zeros((1000, 1000)) + fam = d.add_relation_family('Drug-Gene', 1, 0, True) - fam.add_relation_type('Target', 1, 0, dummy_0) + fam.add_relation_type('Target', dummy_0) + fam = d.add_relation_family('Gene-Gene', 0, 0, True) - fam.add_relation_type('Interaction', 0, 0, dummy_3) + fam.add_relation_type('Interaction', dummy_3) + fam = d.add_relation_family('Drug-Drug', 1, 1, True) - fam.add_relation_type('Side Effect: Nausea', 1, 1, dummy_2) - fam.add_relation_type('Side Effect: Infertility', 1, 1, dummy_2) - fam.add_relation_type('Side Effect: Death', 1, 1, dummy_2) + fam.add_relation_type('Side Effect: Nausea', dummy_2) + fam.add_relation_type('Side Effect: Infertility', dummy_2) + fam.add_relation_type('Side Effect: Death', dummy_2) + print(d) @@ -40,19 +113,19 @@ def test_data_02(): fam = d.add_relation_family('Drug-Gene', 1, 0, True) with pytest.raises(ValueError): - fam.add_relation_type('Target', 1, 0, dummy_1) + fam.add_relation_type('Target', dummy_1) fam = d.add_relation_family('Gene-Gene', 0, 0, True) with pytest.raises(ValueError): - fam.add_relation_type('Interaction', 0, 0, dummy_2) + fam.add_relation_type('Interaction', dummy_2) fam = d.add_relation_family('Drug-Drug', 1, 1, True) with pytest.raises(ValueError): - fam.add_relation_type('Side Effect: Nausea', 1, 1, dummy_3) + fam.add_relation_type('Side Effect: Nausea', dummy_3) with pytest.raises(ValueError): - fam.add_relation_type('Side Effect: Infertility', 1, 1, dummy_3) + fam.add_relation_type('Side Effect: Infertility', dummy_3) with pytest.raises(ValueError): - fam.add_relation_type('Side Effect: Death', 1, 1, dummy_3) + fam.add_relation_type('Side Effect: Death', dummy_3) print(d) @@ -62,5 +135,5 @@ def test_data_03(): d.add_node_type('Drug', 100) fam = d.add_relation_family('Drug-Gene', 1, 0, True) with pytest.raises(ValueError): - fam.add_relation_type('Target', 1, 0, None) + fam.add_relation_type('Target', None) print(d) diff --git a/tests/icosagon/test_declayer.py b/tests/icosagon/test_declayer.py index bab9a25..14b531b 100644 --- a/tests/icosagon/test_declayer.py +++ b/tests/icosagon/test_declayer.py @@ -22,7 +22,7 @@ def test_decode_layer_01(): d.add_node_type('Dummy', 100) fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) - fam.add_relation_type('Dummy Relation 1', 0, 0, + fam.add_relation_type('Dummy Relation 1', torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) diff --git a/tests/icosagon/test_decode.py b/tests/icosagon/test_decode.py index ee0eaf2..f52a3c7 100644 --- a/tests/icosagon/test_decode.py +++ b/tests/icosagon/test_decode.py @@ -151,7 +151,6 @@ def test_is_inner_product_symmetric_01(): res_1 = [ dec(repr_1, repr_2, k) for k in range(7) ] res_2 = [ dec(repr_2, repr_1, k) for k in range(7) ] - assert isinstance(res_1, list) assert isinstance(res_2, list) diff --git a/tests/icosagon/test_input.py b/tests/icosagon/test_input.py index 1ac2e3d..4f50912 100644 --- a/tests/icosagon/test_input.py +++ b/tests/icosagon/test_input.py @@ -11,15 +11,15 @@ def _some_data(): d.add_node_type('Drug', 100) fam = d.add_relation_family('Drug-Gene', 1, 0, False) - fam.add_relation_type('Target', 1, 0, torch.rand(100, 1000)) + fam.add_relation_type('Target', torch.rand(100, 1000)) fam = d.add_relation_family('Gene-Gene', 0, 0, False) - fam.add_relation_type('Interaction', 0, 0, torch.rand(1000, 1000)) + fam.add_relation_type('Interaction', torch.rand(1000, 1000)) fam = d.add_relation_family('Drug-Drug', 1, 1, False) - fam.add_relation_type('Side Effect: Nausea', 1, 1, torch.rand(100, 100)) - fam.add_relation_type('Side Effect: Infertility', 1, 1, torch.rand(100, 100)) - fam.add_relation_type('Side Effect: Death', 1, 1, torch.rand(100, 100)) + fam.add_relation_type('Side Effect: Nausea', torch.rand(100, 100)) + fam.add_relation_type('Side Effect: Infertility', torch.rand(100, 100)) + fam.add_relation_type('Side Effect: Death', torch.rand(100, 100)) return d -- 2.26.2 From f3371d996d9565fff13c98a19308f1a4075f9d6b Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 12 Jun 2020 17:55:19 +0200 Subject: [PATCH 088/227] Add tests for decoders on empty inputs. --- tests/icosagon/test_decode.py | 52 +++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/icosagon/test_decode.py b/tests/icosagon/test_decode.py index f52a3c7..96ca3d7 100644 --- a/tests/icosagon/test_decode.py +++ b/tests/icosagon/test_decode.py @@ -158,3 +158,55 @@ def test_is_inner_product_symmetric_01(): for i in range(len(res_1)): assert torch.all(res_1[i] - res_2[i] < 0.000001) + + +def test_empty_dedicom_decoder_01(): + repr_ = torch.rand(0, 32) + dec = DEDICOMDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + + res = [ dec(repr_, repr_, k) for k in range(7) ] + + assert isinstance(res, list) + + for i in range(len(res)): + assert res[i].shape == (0,) + + +def test_empty_dist_mult_decoder_01(): + repr_ = torch.rand(0, 32) + dec = DistMultDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + + res = [ dec(repr_, repr_, k) for k in range(7) ] + + assert isinstance(res, list) + + for i in range(len(res)): + assert res[i].shape == (0,) + + +def test_empty_bilinear_decoder_01(): + repr_ = torch.rand(0, 32) + dec = BilinearDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + + res = [ dec(repr_, repr_, k) for k in range(7) ] + + assert isinstance(res, list) + + for i in range(len(res)): + assert res[i].shape == (0,) + + +def test_empty_inner_product_decoder_01(): + repr_ = torch.rand(0, 32) + dec = InnerProductDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + + res = [ dec(repr_, repr_, k) for k in range(7) ] + + assert isinstance(res, list) + + for i in range(len(res)): + assert res[i].shape == (0,) -- 2.26.2 From bf913e4e4ba9aed6cff6c30397f8590015bcbb55 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 12 Jun 2020 18:55:47 +0200 Subject: [PATCH 089/227] Add icosagon-reltype-rules. --- docs/icosagon-reltype-rules.svg | 636 ++++++++++++++++++++++++++++++++ 1 file changed, 636 insertions(+) create mode 100644 docs/icosagon-reltype-rules.svg diff --git a/docs/icosagon-reltype-rules.svg b/docs/icosagon-reltype-rules.svg new file mode 100644 index 0000000..bda866a --- /dev/null +++ b/docs/icosagon-reltype-rules.svg @@ -0,0 +1,636 @@ + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + adjacency_matrix + adjacency_matrix_backward + is_symmetric + + + + + + + + + + + + + X + X + X + + X + + + + + + X + X + X + + X + + + + + + Forbidden + + + + + X + X + X + X + + + + + + + + + + + auto + one_node_type + two_node_types + + Forbidden + + Forbidden + + Forbidden + + + + + + + + + + -- 2.26.2 From fe6c6598f8466e6074d3ccc0bc218c04b32a3a69 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 12 Jun 2020 21:09:04 +0200 Subject: [PATCH 090/227] Add module nesting tests. --- tests/icosagon/test_convlayer.py | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/icosagon/test_convlayer.py b/tests/icosagon/test_convlayer.py index 145ad95..1b6fdba 100644 --- a/tests/icosagon/test_convlayer.py +++ b/tests/icosagon/test_convlayer.py @@ -183,3 +183,48 @@ def test_decagon_layer_05(): assert len(out_d_layer) == 1 assert torch.all(out_d_layer[0] == out_multi_dgca) + + +class Dummy1(torch.nn.Module): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.whatever = torch.nn.Parameter(torch.rand((10, 10))) + + +class Dummy2(torch.nn.Module): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.dummy_1 = Dummy1() + + +class Dummy3(torch.nn.Module): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.dummy_1 = [ Dummy1() ] + + +class Dummy4(torch.nn.Module): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.dummy_1 = torch.nn.ModuleList([ Dummy1() ]) + + +def test_module_nesting_01(): + device = torch.device('cuda:0') + dummy_2 = Dummy2() + dummy_2 = dummy_2.to(device) + assert dummy_2.dummy_1.whatever.device == device + + +def test_module_nesting_02(): + device = torch.device('cuda:0') + dummy_3 = Dummy3() + dummy_3 = dummy_3.to(device) + assert dummy_3.dummy_1[0].whatever.device != device + + +def test_module_nesting_03(): + device = torch.device('cuda:0') + dummy_4 = Dummy4() + dummy_4 = dummy_4.to(device) + assert dummy_4.dummy_1[0].whatever.device == device -- 2.26.2 From 55a5f3a2d2a742edaa064c17a21497c053ea7057 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 12 Jun 2020 21:38:00 +0200 Subject: [PATCH 091/227] Use torch.nn.ModuleList in convlayer. --- src/icosagon/convlayer.py | 21 ++++++++---- tests/icosagon/test_convlayer.py | 56 ++++++++++++++++++++++++++++++-- 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/src/icosagon/convlayer.py b/src/icosagon/convlayer.py index cdaced6..e98b55e 100644 --- a/src/icosagon/convlayer.py +++ b/src/icosagon/convlayer.py @@ -9,10 +9,16 @@ from collections import defaultdict from dataclasses import dataclass -@dataclass -class Convolutions(object): +class Convolutions(torch.nn.Module): node_type_column: int - convolutions: List[DropoutGraphConvActivation] + convolutions: torch.nn.ModuleList # [DropoutGraphConvActivation] + + def __init__(self, node_type_column: int, + convolutions: torch.nn.ModuleList, **kwargs): + + super().__init__(**kwargs) + self.node_type_column = node_type_column + self.convolutions = convolutions class DecagonLayer(torch.nn.Module): @@ -51,7 +57,7 @@ class DecagonLayer(torch.nn.Module): self.build() def build_fam_one_node_type(self, fam): - convolutions = [] + convolutions = torch.nn.ModuleList() for r in fam.relation_types: conv = DropoutGraphConvActivation(self.input_dim[fam.node_type_column], @@ -63,8 +69,8 @@ class DecagonLayer(torch.nn.Module): Convolutions(fam.node_type_column, convolutions)) def build_fam_two_node_types(self, fam) -> None: - convolutions_row = [] - convolutions_column = [] + convolutions_row = torch.nn.ModuleList() + convolutions_column = torch.nn.ModuleList() for r in fam.relation_types: if r.adjacency_matrix is not None: @@ -92,7 +98,8 @@ class DecagonLayer(torch.nn.Module): self.build_fam_two_node_types(fam) def build(self): - self.next_layer_repr = [ [] for _ in range(len(self.data.node_types)) ] + self.next_layer_repr = torch.nn.ModuleList([ + torch.nn.ModuleList() for _ in range(len(self.data.node_types)) ]) for fam in self.data.relation_families: self.build_family(fam) diff --git a/tests/icosagon/test_convlayer.py b/tests/icosagon/test_convlayer.py index 1b6fdba..96fb225 100644 --- a/tests/icosagon/test_convlayer.py +++ b/tests/icosagon/test_convlayer.py @@ -77,10 +77,10 @@ def test_decagon_layer_03(): for i in range(2): assert len(d_layer.next_layer_repr[i]) == 2 - assert isinstance(d_layer.next_layer_repr[i], list) + assert isinstance(d_layer.next_layer_repr[i], torch.nn.ModuleList) assert isinstance(d_layer.next_layer_repr[i][0], Convolutions) assert isinstance(d_layer.next_layer_repr[i][0].node_type_column, int) - assert isinstance(d_layer.next_layer_repr[i][0].convolutions, list) + assert isinstance(d_layer.next_layer_repr[i][0].convolutions, torch.nn.ModuleList) assert all([ isinstance(dgca, DropoutGraphConvActivation) \ for dgca in d_layer.next_layer_repr[i][0].convolutions @@ -209,7 +209,28 @@ class Dummy4(torch.nn.Module): self.dummy_1 = torch.nn.ModuleList([ Dummy1() ]) +class Dummy5(torch.nn.Module): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.dummy_1 = [ torch.nn.ModuleList([ Dummy1() ]) ] + + +class Dummy6(torch.nn.Module): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.dummy_1 = torch.nn.ModuleList([ torch.nn.ModuleList([ Dummy1() ]) ]) + + +class Dummy7(torch.nn.Module): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.dummy_1 = torch.nn.ModuleList([ torch.nn.ModuleList() ]) + self.dummy_1[0].append(Dummy1()) + + def test_module_nesting_01(): + if torch.cuda.device_count() == 0: + pytest.skip('No CUDA support on this host') device = torch.device('cuda:0') dummy_2 = Dummy2() dummy_2 = dummy_2.to(device) @@ -217,6 +238,8 @@ def test_module_nesting_01(): def test_module_nesting_02(): + if torch.cuda.device_count() == 0: + pytest.skip('No CUDA support on this host') device = torch.device('cuda:0') dummy_3 = Dummy3() dummy_3 = dummy_3.to(device) @@ -224,7 +247,36 @@ def test_module_nesting_02(): def test_module_nesting_03(): + if torch.cuda.device_count() == 0: + pytest.skip('No CUDA support on this host') device = torch.device('cuda:0') dummy_4 = Dummy4() dummy_4 = dummy_4.to(device) assert dummy_4.dummy_1[0].whatever.device == device + + +def test_module_nesting_04(): + if torch.cuda.device_count() == 0: + pytest.skip('No CUDA support on this host') + device = torch.device('cuda:0') + dummy_5 = Dummy5() + dummy_5 = dummy_5.to(device) + assert dummy_5.dummy_1[0][0].whatever.device != device + + +def test_module_nesting_05(): + if torch.cuda.device_count() == 0: + pytest.skip('No CUDA support on this host') + device = torch.device('cuda:0') + dummy_6 = Dummy6() + dummy_6 = dummy_6.to(device) + assert dummy_6.dummy_1[0][0].whatever.device == device + + +def test_module_nesting_06(): + if torch.cuda.device_count() == 0: + pytest.skip('No CUDA support on this host') + device = torch.device('cuda:0') + dummy_7 = Dummy7() + dummy_7 = dummy_7.to(device) + assert dummy_7.dummy_1[0][0].whatever.device == device -- 2.26.2 From 25e05cf1c2853672ddfc8eaf2a1a4c3123d88283 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 12 Jun 2020 21:49:44 +0200 Subject: [PATCH 092/227] Use torch.nn.Parameter(List) in decode. --- src/icosagon/decode.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/icosagon/decode.py b/src/icosagon/decode.py index b69ec97..00df8b2 100644 --- a/src/icosagon/decode.py +++ b/src/icosagon/decode.py @@ -20,11 +20,11 @@ class DEDICOMDecoder(torch.nn.Module): self.keep_prob = keep_prob self.activation = activation - self.global_interaction = init_glorot(input_dim, input_dim) - self.local_variation = [ - torch.flatten(init_glorot(input_dim, 1)) \ + self.global_interaction = torch.nn.Parameter(init_glorot(input_dim, input_dim)) + self.local_variation = torch.nn.ParameterList([ + torch.nn.Parameter(torch.flatten(init_glorot(input_dim, 1))) \ for _ in range(num_relation_types) - ] + ]) def forward(self, inputs_row, inputs_col, relation_index): inputs_row = dropout(inputs_row, self.keep_prob) @@ -53,10 +53,10 @@ class DistMultDecoder(torch.nn.Module): self.keep_prob = keep_prob self.activation = activation - self.relation = [ - torch.flatten(init_glorot(input_dim, 1)) \ + self.relation = torch.nn.ParameterList([ + torch.nn.Parameter(torch.flatten(init_glorot(input_dim, 1))) \ for _ in range(num_relation_types) - ] + ]) def forward(self, inputs_row, inputs_col, relation_index): inputs_row = dropout(inputs_row, self.keep_prob) @@ -83,10 +83,10 @@ class BilinearDecoder(torch.nn.Module): self.keep_prob = keep_prob self.activation = activation - self.relation = [ - init_glorot(input_dim, input_dim) \ + self.relation = torch.nn.ParameterList([ + torch.nn.Parameter(init_glorot(input_dim, input_dim)) \ for _ in range(num_relation_types) - ] + ]) def forward(self, inputs_row, inputs_col, relation_index): inputs_row = dropout(inputs_row, self.keep_prob) -- 2.26.2 From 093d378c6aabe8ed4e3da4faf0771adfb0b54651 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 15 Jun 2020 11:38:14 +0200 Subject: [PATCH 093/227] Fix test_decode_layer_02. --- tests/icosagon/test_declayer.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/icosagon/test_declayer.py b/tests/icosagon/test_declayer.py index 14b531b..17b74f9 100644 --- a/tests/icosagon/test_declayer.py +++ b/tests/icosagon/test_declayer.py @@ -55,22 +55,24 @@ def test_decode_layer_01(): def test_decode_layer_02(): d = Data() d.add_node_type('Dummy', 100) - d.add_relation_type('Dummy Relation 1', 0, 0, + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Relation 1', torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) in_layer = OneHotInputLayer(d) d_layer = DecagonLayer(in_layer.output_dim, 32, d) - dec_layer = DecodeLayer(input_dim=d_layer.output_dim, data=prep_d, keep_prob=1., - decoder_class=DEDICOMDecoder, activation=lambda x: x) + dec_layer = DecodeLayer(input_dim=d_layer.output_dim, data=prep_d, + keep_prob=1., activation=lambda x: x) seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) - pred_adj_matrices = seq(None) + pred = seq(None) - assert isinstance(pred_adj_matrices, dict) - assert len(pred_adj_matrices) == 1 - assert isinstance(pred_adj_matrices[0, 0], list) - assert len(pred_adj_matrices[0, 0]) == 1 + assert isinstance(pred, Predictions) + assert len(pred.relation_families) == 1 + assert isinstance(pred.relation_families[0], RelationFamilyPredictions) + assert isinstance(pred.relation_families[0].relation_types, list) + assert len(pred.relation_families[0].relation_types) == 1 def test_decode_layer_03(): -- 2.26.2 From fc11f052c4dd4256ef898a7922d5c57cc4734a2b Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 15 Jun 2020 11:42:54 +0200 Subject: [PATCH 094/227] Fix test_decode_layer_03. --- tests/icosagon/test_declayer.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/icosagon/test_declayer.py b/tests/icosagon/test_declayer.py index 17b74f9..26eb3cb 100644 --- a/tests/icosagon/test_declayer.py +++ b/tests/icosagon/test_declayer.py @@ -79,23 +79,24 @@ def test_decode_layer_03(): d = Data() d.add_node_type('Dummy 1', 100) d.add_node_type('Dummy 2', 100) - d.add_relation_type('Dummy Relation 1', 0, 1, + fam = d.add_relation_family('Dummy 1-Dummy 2', 0, 1, True) + fam.add_relation_type('Dummy Relation 1', torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) in_layer = OneHotInputLayer(d) d_layer = DecagonLayer(in_layer.output_dim, 32, d) - dec_layer = DecodeLayer(input_dim=d_layer.output_dim, data=prep_d, keep_prob=1., - decoder_class={(0, 1): DEDICOMDecoder}, activation=lambda x: x) + dec_layer = DecodeLayer(input_dim=d_layer.output_dim, data=prep_d, + keep_prob=1., activation=lambda x: x) seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) - pred_adj_matrices = seq(None) - assert isinstance(pred_adj_matrices, dict) - assert len(pred_adj_matrices) == 2 - assert isinstance(pred_adj_matrices[0, 1], list) - assert isinstance(pred_adj_matrices[1, 0], list) - assert len(pred_adj_matrices[0, 1]) == 1 - assert len(pred_adj_matrices[1, 0]) == 1 + pred = seq(None) + assert isinstance(pred, Predictions) + assert len(pred.relation_families) == 1 + assert isinstance(pred.relation_families[0], RelationFamilyPredictions) + assert isinstance(pred.relation_families[0].relation_types, list) + assert len(pred.relation_families[0].relation_types) == 1 + assert isinstance(pred.relation_families[0].relation_types[0], RelationPredictions) def test_decode_layer_04(): -- 2.26.2 From 5ab9231efc1456eec2ec92d89506cfe6d7895f10 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 15 Jun 2020 11:53:24 +0200 Subject: [PATCH 095/227] Fix test_decode_layer_04. --- tests/icosagon/test_declayer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/icosagon/test_declayer.py b/tests/icosagon/test_declayer.py index 26eb3cb..cd903f1 100644 --- a/tests/icosagon/test_declayer.py +++ b/tests/icosagon/test_declayer.py @@ -102,17 +102,17 @@ def test_decode_layer_03(): def test_decode_layer_04(): d = Data() d.add_node_type('Dummy', 100) - assert len(d.relation_types[0, 0]) == 0 + assert len(d.relation_families) == 0 prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) in_layer = OneHotInputLayer(d) d_layer = DecagonLayer(in_layer.output_dim, 32, d) - dec_layer = DecodeLayer(input_dim=d_layer.output_dim, data=prep_d, keep_prob=1., - decoder_class=DEDICOMDecoder, activation=lambda x: x) + dec_layer = DecodeLayer(input_dim=d_layer.output_dim, data=prep_d, + keep_prob=1., activation=lambda x: x) seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) - pred_adj_matrices = seq(None) + pred = seq(None) - assert isinstance(pred_adj_matrices, dict) - assert len(pred_adj_matrices) == 0 + assert isinstance(pred, Predictions) + assert len(pred.relation_families) == 0 -- 2.26.2 From b92c7a0b5d2863b1e0eed7e7c07585523ce50726 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 15 Jun 2020 17:01:09 +0200 Subject: [PATCH 096/227] Add test_decode_layer_05. --- tests/icosagon/test_declayer.py | 110 +++++++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 1 deletion(-) diff --git a/tests/icosagon/test_declayer.py b/tests/icosagon/test_declayer.py index cd903f1..0a3c617 100644 --- a/tests/icosagon/test_declayer.py +++ b/tests/icosagon/test_declayer.py @@ -5,12 +5,14 @@ from icosagon.input import OneHotInputLayer +from icosagon.convolve import DropoutGraphConvActivation from icosagon.convlayer import DecagonLayer from icosagon.declayer import DecodeLayer, \ Predictions, \ RelationFamilyPredictions, \ RelationPredictions -from icosagon.decode import DEDICOMDecoder +from icosagon.decode import DEDICOMDecoder, \ + InnerProductDecoder from icosagon.data import Data from icosagon.trainprep import prepare_training, \ TrainValTest @@ -116,3 +118,109 @@ def test_decode_layer_04(): assert isinstance(pred, Predictions) assert len(pred.relation_families) == 0 + + +def test_decode_layer_05(): + d = Data() + d.add_node_type('Dummy', 10) + mat = torch.rand((10, 10)) + mat = (mat + mat.transpose(0, 1)) / 2 + mat = mat.round() + fam = d.add_relation_family('Dummy-Dummy', 0, 0, True, + decoder_class=InnerProductDecoder) + fam.add_relation_type('Dummy Rel', mat.to_sparse()) + prep_d = prepare_training(d, TrainValTest(1., 0., 0.)) + + in_layer = OneHotInputLayer(d) + conv_layer = DecagonLayer(in_layer.output_dim, 32, prep_d, + rel_activation=lambda x: x, layer_activation=lambda x: x) + dec_layer = DecodeLayer(conv_layer.output_dim, prep_d, + keep_prob=1., activation=lambda x: x) + seq = torch.nn.Sequential(in_layer, conv_layer, dec_layer) + + pred = seq(None) + rel_pred = pred.relation_families[0].relation_types[0] + + for edge_type in ['edges_pos', 'edges_neg', 'edges_back_pos', 'edges_back_neg']: + edge_pred = getattr(rel_pred, edge_type) + assert isinstance(edge_pred, TrainValTest) + for part_type in ['train', 'val', 'test']: + part_pred = getattr(edge_pred, part_type) + assert isinstance(part_pred, torch.Tensor) + assert len(part_pred.shape) == 1 + print(edge_type, part_type, part_pred.shape) + if (edge_type, part_type) not in [('edges_pos', 'train'), ('edges_neg', 'train')]: + assert part_pred.shape[0] == 0 + else: + assert part_pred.shape[0] > 0 + + prep_rel = prep_d.relation_families[0].relation_types[0] + assert len(rel_pred.edges_pos.train) == len(prep_rel.edges_pos.train) + assert len(rel_pred.edges_neg.train) == len(prep_rel.edges_neg.train) + + assert len(prep_rel.edges_pos.train) == torch.sum(mat) + + # print('Predictions for positive edges:') + # print(rel_pred.edges_pos.train) + # print('Predictions for negative edges:') + # print(rel_pred.edges_neg.train) + + repr_in = in_layer(None) + assert isinstance(repr_in, list) + assert len(repr_in) == 1 + assert isinstance(repr_in[0], torch.Tensor) + assert torch.all(repr_in[0].to_dense() == torch.eye(10)) + + assert len(conv_layer.next_layer_repr[0]) == 1 + assert len(conv_layer.next_layer_repr[0][0].convolutions) == 1 + assert conv_layer.rel_activation(0) == 0 + assert conv_layer.rel_activation(1) == 1 + assert conv_layer.rel_activation(-1) == -1 + assert conv_layer.layer_activation(0) == 0 + assert conv_layer.layer_activation(1) == 1 + assert conv_layer.layer_activation(-1) == -1 + + graph_conv = conv_layer.next_layer_repr[0][0].convolutions[0] + assert isinstance(graph_conv, DropoutGraphConvActivation) + assert graph_conv.activation(0) == 0 + assert graph_conv.activation(1) == 1 + assert graph_conv.activation(-1) == -1 + weight = graph_conv.graph_conv.weight + adj_mat = prep_d.relation_families[0].relation_types[0].adjacency_matrix + repr_conv = torch.sparse.mm(repr_in[0], weight) + repr_conv = torch.mm(adj_mat, repr_conv) + repr_conv = torch.nn.functional.normalize(repr_conv, p=2, dim=1) + repr_conv_expect = conv_layer(repr_in)[0] + print('repr_conv:\n', repr_conv) + # print(repr_conv_expect) + assert torch.all(repr_conv == repr_conv_expect) + assert repr_conv.shape[1] == 32 + + dec = InnerProductDecoder(32, 1, keep_prob=1., activation=lambda x: x) + x, y = torch.meshgrid(torch.arange(0, 10), torch.arange(0, 10)) + x = x.flatten() + y = y.flatten() + repr_dec_expect = dec(repr_conv[x], repr_conv[y], 0) + repr_dec_expect = repr_dec_expect.view(10, 10) + + + repr_dec = torch.mm(repr_conv, torch.transpose(repr_conv, 0, 1)) + # repr_dec = torch.flatten(repr_dec) + # repr_dec -= torch.eye(10) + #repr_dec_expect = torch.zeros((10, 10)) + #x = prep_d.relation_families[0].relation_types[0].edges_pos.train + #repr_dec_expect[x[:, 0], x[:, 1]] = pred.relation_families[0].relation_types[0].edges_pos.train + #x = prep_d.relation_families[0].relation_types[0].edges_neg.train + #repr_dec_expect[x[:, 0], x[:, 1]] = pred.relation_families[0].relation_types[0].edges_neg.train + print(repr_dec) + print(repr_dec_expect) + assert torch.all(torch.abs(repr_dec - repr_dec_expect) < 0.000001) + + #print(prep_rel.edges_pos.train) + #print(prep_rel.edges_neg.train) + + # assert isinstance(edge_pred.train) + # assert isinstance(rel_pred.edges_pos, TrainValTest) + # assert isinstance(rel_pred.edges_neg, TrainValTest) + # assert isinstance(rel_pred.edges_back_pos, TrainValTest) + # assert isinstance(rel_pred.edges_back_neg, TrainValTest) -- 2.26.2 From dc8c51a8e6c0198ec84f274f7222a601222d6749 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 15 Jun 2020 17:12:52 +0200 Subject: [PATCH 097/227] Ammend test_decode_layer_05. --- tests/icosagon/test_declayer.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/icosagon/test_declayer.py b/tests/icosagon/test_declayer.py index 0a3c617..216b1ed 100644 --- a/tests/icosagon/test_declayer.py +++ b/tests/icosagon/test_declayer.py @@ -203,17 +203,25 @@ def test_decode_layer_05(): repr_dec_expect = dec(repr_conv[x], repr_conv[y], 0) repr_dec_expect = repr_dec_expect.view(10, 10) - repr_dec = torch.mm(repr_conv, torch.transpose(repr_conv, 0, 1)) # repr_dec = torch.flatten(repr_dec) # repr_dec -= torch.eye(10) - #repr_dec_expect = torch.zeros((10, 10)) - #x = prep_d.relation_families[0].relation_types[0].edges_pos.train - #repr_dec_expect[x[:, 0], x[:, 1]] = pred.relation_families[0].relation_types[0].edges_pos.train - #x = prep_d.relation_families[0].relation_types[0].edges_neg.train - #repr_dec_expect[x[:, 0], x[:, 1]] = pred.relation_families[0].relation_types[0].edges_neg.train + assert torch.all(torch.abs(repr_dec - repr_dec_expect) < 0.000001) + + repr_dec_expect = torch.zeros((10, 10)) + x = prep_d.relation_families[0].relation_types[0].edges_pos.train + repr_dec_expect[x[:, 0], x[:, 1]] = pred.relation_families[0].relation_types[0].edges_pos.train + x = prep_d.relation_families[0].relation_types[0].edges_neg.train + repr_dec_expect[x[:, 0], x[:, 1]] = pred.relation_families[0].relation_types[0].edges_neg.train print(repr_dec) print(repr_dec_expect) + + repr_dec = torch.zeros((10, 10)) + x = prep_d.relation_families[0].relation_types[0].edges_pos.train + repr_dec[x[:, 0], x[:, 1]] = dec(repr_conv[x[:, 0]], repr_conv[x[:, 1]], 0) + x = prep_d.relation_families[0].relation_types[0].edges_neg.train + repr_dec[x[:, 0], x[:, 1]] = dec(repr_conv[x[:, 0]], repr_conv[x[:, 1]], 0) + assert torch.all(torch.abs(repr_dec - repr_dec_expect) < 0.000001) #print(prep_rel.edges_pos.train) -- 2.26.2 From ff4fb96bed038eaca1654a24a1772f1ce925519d Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 15 Jun 2020 17:56:19 +0200 Subject: [PATCH 098/227] Add CrossEntropyLoss and test_cross_entropy_loss_01(). --- src/icosagon/loss.py | 44 +++++++++++++++++++++++++++++++++++++ tests/icosagon/test_loss.py | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 src/icosagon/loss.py create mode 100644 tests/icosagon/test_loss.py diff --git a/src/icosagon/loss.py b/src/icosagon/loss.py new file mode 100644 index 0000000..eb2e0fe --- /dev/null +++ b/src/icosagon/loss.py @@ -0,0 +1,44 @@ +import torch +from icosagon.trainprep import PreparedData +from icosagon.declayer import Predictions + + +class CrossEntropyLoss(torch.nn.Module): + def __init__(self, data: PreparedData, partition_type: str = 'train', + reduction: str = 'sum', **kwargs) -> None: + + super().__init__(**kwargs) + + if not isinstance(data, PreparedData): + raise TypeError('data must be an instance of PreparedData') + + if partition_type not in ['train', 'val', 'test']: + raise ValueError('partition_type must be set to train, val or test') + + if reduction not in ['sum', 'mean']: + raise ValueError('reduction must be set to sum or mean') + + self.data = data + self.partition_type = partition_type + self.reduction = reduction + + def forward(self, pred: Predictions) -> torch.Tensor: + input = [] + target = [] + for fam in pred.relation_families: + for rel in fam.relation_types: + for edge_type in ['edges_pos', 'edges_back_pos']: + x = getattr(getattr(rel, edge_type), self.partition_type) + assert len(x.shape) == 1 + input.append(x) + target.append(torch.ones_like(x)) + for edge_type in ['edges_neg', 'edges_back_neg']: + x = getattr(getattr(rel, edge_type), self.partition_type) + assert len(x.shape) == 1 + input.append(x) + target.append(torch.zeros_like(x)) + input = torch.cat(input, dim=0) + target = torch.cat(target, dim=0) + res = torch.nn.functional.binary_cross_entropy(input, target, + reduction=self.reduction) + return res diff --git a/tests/icosagon/test_loss.py b/tests/icosagon/test_loss.py new file mode 100644 index 0000000..07e5e5c --- /dev/null +++ b/tests/icosagon/test_loss.py @@ -0,0 +1,40 @@ +from icosagon.loss import CrossEntropyLoss +from icosagon.declayer import Predictions, \ + RelationFamilyPredictions, \ + RelationPredictions +from icosagon.data import Data +from icosagon.trainprep import prepare_training, \ + TrainValTest +import torch + + +def test_cross_entropy_loss_01(): + d = Data() + d.add_node_type('Dummy', 5) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Rel', torch.tensor([ + [0, 1, 0, 0, 0], + [1, 0, 0, 0, 0], + [0, 0, 0, 1, 0], + [0, 0, 0, 0, 1], + [0, 1, 0, 0, 0] + ], dtype=torch.float32)) + + prep_d = prepare_training(d, TrainValTest(1., 0., 0.)) + + rel_pred = RelationPredictions( + TrainValTest(torch.tensor([1, 0, 1, 0, 1], dtype=torch.float32), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)) + ) + fam_pred = RelationFamilyPredictions([ rel_pred ]) + pred = Predictions([ fam_pred ]) + + loss = CrossEntropyLoss(prep_d) + print('loss: %.7f' % loss(pred)) + assert torch.abs(loss(pred) - 55.262043) < 0.000001 + + loss = CrossEntropyLoss(prep_d, reduction='mean') + print('loss: %.7f' % loss(pred)) + assert torch.abs(loss(pred) - 11.0524082) < 0.000001 -- 2.26.2 From 4ab4bd6eb21553ceb4af5234fd38714074ffb08a Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 15 Jun 2020 18:03:34 +0200 Subject: [PATCH 099/227] Add some asserts to test_cross_entropy_loss_01(). --- tests/icosagon/test_loss.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/icosagon/test_loss.py b/tests/icosagon/test_loss.py index 07e5e5c..4f1edde 100644 --- a/tests/icosagon/test_loss.py +++ b/tests/icosagon/test_loss.py @@ -22,6 +22,12 @@ def test_cross_entropy_loss_01(): prep_d = prepare_training(d, TrainValTest(1., 0., 0.)) + assert len(prep_d.relation_families) == 1 + assert len(prep_d.relation_families[0].relation_types) == 1 + assert len(prep_d.relation_families[0].relation_types[0].edges_pos.train) == 5 + assert len(prep_d.relation_families[0].relation_types[0].edges_pos.val) == 0 + assert len(prep_d.relation_families[0].relation_types[0].edges_pos.test) == 0 + rel_pred = RelationPredictions( TrainValTest(torch.tensor([1, 0, 1, 0, 1], dtype=torch.float32), torch.zeros(0), torch.zeros(0)), TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)), -- 2.26.2 From a9f14d14a80888fe2d0b644f3f52dcc188132bed Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 16 Jun 2020 15:11:25 +0200 Subject: [PATCH 100/227] Add PredictionsBatch and test_predictions_btch_01(). --- src/icosagon/batch.py | 39 ++++++++++++++++++++++++++++++ tests/icosagon/test_batch.py | 46 ++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 src/icosagon/batch.py create mode 100644 tests/icosagon/test_batch.py diff --git a/src/icosagon/batch.py b/src/icosagon/batch.py new file mode 100644 index 0000000..275cac3 --- /dev/null +++ b/src/icosagon/batch.py @@ -0,0 +1,39 @@ +from icosagon.declayer import Predictions +import torch + + +class PredictionsBatch(object): + def __init__(self, pred: Predictions, part_type: str = 'train', + batch_size: int = 100) -> None: + + if not isinstance(pred, Predictions): + raise TypeError('pred must be an instance of Predictions') + + if part_type not in ['train', 'val', 'test']: + raise ValueError('part_type must be set to train, val or test') + + batch_size = int(batch_size) + + self.predictions = pred + self.part_type = part_type + self.batch_size = batch_size + + def __iter__(self): + edge_types = [('edges_pos', 1), ('edges_neg', 0), + ('edges_back_pos', 1), ('edges_back_neg', 0)] + + input = [] + target = [] + + for fam in self.predictions.relation_families: + for rel in fam.relation_types: + for (et, tgt) in edge_types: + edge_pred = getattr(getattr(rel, et), self.part_type) + input.append(edge_pred) + target.append(torch.ones_like(edge_pred) * tgt) + + input = torch.cat(input) + target = torch.cat(target) + + for i in range(0, len(input), self.batch_size): + yield (input[i:i+self.batch_size], target[i:i+self.batch_size]) diff --git a/tests/icosagon/test_batch.py b/tests/icosagon/test_batch.py new file mode 100644 index 0000000..fa6a4a9 --- /dev/null +++ b/tests/icosagon/test_batch.py @@ -0,0 +1,46 @@ +from icosagon.batch import PredictionsBatch +from icosagon.declayer import Predictions, \ + RelationPredictions, \ + RelationFamilyPredictions +from icosagon.trainprep import prepare_training, \ + TrainValTest +from icosagon.data import Data +import torch + + +def test_predictions_batch_01(): + d = Data() + d.add_node_type('Dummy', 5) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Rel', torch.tensor([ + [0, 1, 0, 0, 0], + [1, 0, 0, 0, 0], + [0, 0, 0, 1, 0], + [0, 0, 0, 0, 1], + [0, 1, 0, 0, 0] + ], dtype=torch.float32)) + + prep_d = prepare_training(d, TrainValTest(1., 0., 0.)) + + assert len(prep_d.relation_families) == 1 + assert len(prep_d.relation_families[0].relation_types) == 1 + assert len(prep_d.relation_families[0].relation_types[0].edges_pos.train) == 5 + assert len(prep_d.relation_families[0].relation_types[0].edges_neg.train) == 5 + assert len(prep_d.relation_families[0].relation_types[0].edges_pos.val) == 0 + assert len(prep_d.relation_families[0].relation_types[0].edges_pos.test) == 0 + + rel_pred = RelationPredictions( + TrainValTest(torch.tensor([1, 0, 1, 0, 1], dtype=torch.float32), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)) + ) + fam_pred = RelationFamilyPredictions([ rel_pred ]) + pred = Predictions([ fam_pred ]) + + batch = PredictionsBatch(pred, part_type='train', batch_size=1) + count = 0 + for (input, target) in batch: + count += 1 + + assert count == 10 -- 2.26.2 From 72356545aab6ec7589973966a947d28837716d71 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 16 Jun 2020 16:57:22 +0200 Subject: [PATCH 101/227] Remove loss as no longer necessary. --- src/icosagon/{ => unused}/loss.py | 0 tests/icosagon/test_batch.py | 8 +++++++- tests/icosagon/{ => unused}/test_loss.py | 0 3 files changed, 7 insertions(+), 1 deletion(-) rename src/icosagon/{ => unused}/loss.py (100%) rename tests/icosagon/{ => unused}/test_loss.py (100%) diff --git a/src/icosagon/loss.py b/src/icosagon/unused/loss.py similarity index 100% rename from src/icosagon/loss.py rename to src/icosagon/unused/loss.py diff --git a/tests/icosagon/test_batch.py b/tests/icosagon/test_batch.py index fa6a4a9..3d185e4 100644 --- a/tests/icosagon/test_batch.py +++ b/tests/icosagon/test_batch.py @@ -31,7 +31,7 @@ def test_predictions_batch_01(): rel_pred = RelationPredictions( TrainValTest(torch.tensor([1, 0, 1, 0, 1], dtype=torch.float32), torch.zeros(0), torch.zeros(0)), - TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.tensor([1, 0, 1, 0, 1], dtype=torch.float32), torch.zeros(0), torch.zeros(0)), TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)), TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)) ) @@ -40,7 +40,13 @@ def test_predictions_batch_01(): batch = PredictionsBatch(pred, part_type='train', batch_size=1) count = 0 + lst = [] for (input, target) in batch: + assert len(input) == 1 + assert len(target) == 1 + lst.append((input[0], target[0])) count += 1 + assert lst == [ (1, 1), (0, 1), (1, 1), (0, 1), (1, 1), + (1, 0), (0, 0), (1, 0), (0, 0), (1, 0) ] assert count == 10 diff --git a/tests/icosagon/test_loss.py b/tests/icosagon/unused/test_loss.py similarity index 100% rename from tests/icosagon/test_loss.py rename to tests/icosagon/unused/test_loss.py -- 2.26.2 From 908100cf4540831ad99dc8e2a4c44ccf3a21612c Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 16 Jun 2020 17:42:10 +0200 Subject: [PATCH 102/227] Add Model. --- src/icosagon/model.py | 127 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 src/icosagon/model.py diff --git a/src/icosagon/model.py b/src/icosagon/model.py new file mode 100644 index 0000000..ba28bb7 --- /dev/null +++ b/src/icosagon/model.py @@ -0,0 +1,127 @@ +from .data import Data +from typing import List +from .trainprep import prepare_training, \ + TrainValTest +import torch +from .convlayer import DecagonLayer +from .input import OneHotInputLayer +from types import FunctionType +from .declayer import DecodeLayer +from .batch import PredictionsBatch + + +class Model(object): + def __init__(self, data: Data, + layer_dimensions: List[int] = [32, 64], + ratios: TrainValTest = TrainValTest(.8, .1, .1), + keep_prob: float = 1., + rel_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, + layer_activation: Callable[[torch.Tensor], torch.Tensor] = torch.nn.functional.relu, + dec_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, + lr: float = 0.001, + loss = Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = torch.nn.functional.binary_cross_entropy_with_logits, + batch_size: int = 100) -> None: + + if not isinstance(data, Data): + raise TypeError('data must be an instance of Data') + + if not isinstance(layer_sizes, list): + raise TypeError('layer_dimensions must be a list') + + if not isinstance(ratios, TrainValTest): + raise TypeError('ratios must be an instance of TrainValTest') + + keep_prob = float(keep_prob) + + if not isinstance(rel_activation, FunctionType): + raise TypeError('rel_activation must be a function') + + if not isinstance(layer_activation, FunctionType): + raise TypeError('layer_activation must be a function') + + if not isinstance(dec_activation, FunctionType): + raise TypeError('dec_activation must be a function') + + lr = float(lr) + + if not isinstance(loss, FunctionType): + raise TypeError('loss must be a function') + + batch_size = int(batch_size) + + self.data = data + self.layer_dimensions = layer_dimensions + self.ratios = ratios + self.keep_prob = keep_prob + self.rel_activation = rel_activation + self.layer_activation = layer_activation + self.dec_activation = dec_activation + self.lr = lr + self.loss = loss + self.batch_size = batch_size + + self.build() + + def build(self): + self.prep_d = prepare_training(self.data, self.ratios) + + in_layer = OneHotInputLayer(self.prep_d) + last_output_dim = in_layer.output_dim + seq = [ in_layer ] + + for dim in self.layer_dimensions: + conv_layer = DecagonLayer(input_dim = last_output_dim, + output_dim = [ dim ] * len(self.prep_d.node_types), + data = self.prep_d, + keep_prob = self.keep_prob, + rel_activation = self.rel_activation, + layer_activation = self.layer_activation) + last_output_dim = conv_layer.output_dim + seq.append(conv_layer) + + dec_layer = DecodeLayer(input_dim = last_output_dim, + data = self.prep_d, + keep_prob = self.keep_prob, + activation = self.dec_activation) + seq.append(dec_layer) + + seq = torch.nn.Sequential(*seq) + self.seq = seq + + opt = torch.optim.Adam(seq.parameters(), lr=self.lr) + self.opt = opt + + + def run_epoch(self): + pred = self.seq(None) + batch = PredictionsBatch(pred, self.batch_size) + n = len(list(iter(batch))) + loss_sum = 0 + for i in range(n - 1): + self.opt.zero_grad() + pred = self.seq(None) + batch = PredictionsBatch(pred, self.batch_size) + seed = torch.rand(1).item() + rng_state = torch.get_rng_state() + torch.manual_seed(seed) + it = iter(batch) + torch.set_rng_state(rng_state) + for k in range(i): + _ = next(it) + (input, target) = next(it) + loss = self.loss(input, target) + loss.backward() + self.opt.optimize() + loss_sum += loss.detach().cpu().item() + return loss_sum + + + def train(self, max_epochs): + best_loss = None + best_epoch = None + for i in range(max_epochs): + loss = self.run_epoch() + if best_loss is None or loss < best_loss: + best_loss = loss + best_epoch = i + return loss, best_loss, best_epoch -- 2.26.2 From 346cc747a6007c6e9e1ef740439440d23e6be821 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 16 Jun 2020 18:12:44 +0200 Subject: [PATCH 103/227] Add shuffle to PredictionsBatch. --- src/icosagon/batch.py | 10 +++++++++- src/icosagon/model.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/icosagon/batch.py b/src/icosagon/batch.py index 275cac3..9a28712 100644 --- a/src/icosagon/batch.py +++ b/src/icosagon/batch.py @@ -4,7 +4,7 @@ import torch class PredictionsBatch(object): def __init__(self, pred: Predictions, part_type: str = 'train', - batch_size: int = 100) -> None: + batch_size: int = 100, shuffle: bool = False) -> None: if not isinstance(pred, Predictions): raise TypeError('pred must be an instance of Predictions') @@ -14,9 +14,12 @@ class PredictionsBatch(object): batch_size = int(batch_size) + shuffle = bool(shuffle) + self.predictions = pred self.part_type = part_type self.batch_size = batch_size + self.shuffle = shuffle def __iter__(self): edge_types = [('edges_pos', 1), ('edges_neg', 0), @@ -35,5 +38,10 @@ class PredictionsBatch(object): input = torch.cat(input) target = torch.cat(target) + if self.shuffle: + perm = torch.randperm(len(input)) + input = input[perm] + target = target[perm] + for i in range(0, len(input), self.batch_size): yield (input[i:i+self.batch_size], target[i:i+self.batch_size]) diff --git a/src/icosagon/model.py b/src/icosagon/model.py index ba28bb7..4c1cf0d 100644 --- a/src/icosagon/model.py +++ b/src/icosagon/model.py @@ -100,7 +100,7 @@ class Model(object): for i in range(n - 1): self.opt.zero_grad() pred = self.seq(None) - batch = PredictionsBatch(pred, self.batch_size) + batch = PredictionsBatch(pred, self.batch_size, shuffle=True) seed = torch.rand(1).item() rng_state = torch.get_rng_state() torch.manual_seed(seed) -- 2.26.2 From 46bf3b132b26837fa6ca2d0334fe9509edb7bb85 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 16 Jun 2020 19:01:07 +0200 Subject: [PATCH 104/227] Rename test_loss. --- tests/icosagon/unused/{test_loss.py => _test_loss.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/icosagon/unused/{test_loss.py => _test_loss.py} (100%) diff --git a/tests/icosagon/unused/test_loss.py b/tests/icosagon/unused/_test_loss.py similarity index 100% rename from tests/icosagon/unused/test_loss.py rename to tests/icosagon/unused/_test_loss.py -- 2.26.2 From d5779daac3c76a9b8c90a07ed73ff1ac59ec28d3 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 17 Jun 2020 11:30:07 +0200 Subject: [PATCH 105/227] Add test_model_01(). --- src/icosagon/model.py | 11 ++++++++--- tests/icosagon/test_model.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 tests/icosagon/test_model.py diff --git a/src/icosagon/model.py b/src/icosagon/model.py index 4c1cf0d..5d384f0 100644 --- a/src/icosagon/model.py +++ b/src/icosagon/model.py @@ -1,5 +1,6 @@ from .data import Data -from typing import List +from typing import List, \ + Callable from .trainprep import prepare_training, \ TrainValTest import torch @@ -19,13 +20,13 @@ class Model(object): layer_activation: Callable[[torch.Tensor], torch.Tensor] = torch.nn.functional.relu, dec_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, lr: float = 0.001, - loss = Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = torch.nn.functional.binary_cross_entropy_with_logits, + loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = torch.nn.functional.binary_cross_entropy_with_logits, batch_size: int = 100) -> None: if not isinstance(data, Data): raise TypeError('data must be an instance of Data') - if not isinstance(layer_sizes, list): + if not isinstance(layer_dimensions, list): raise TypeError('layer_dimensions must be a list') if not isinstance(ratios, TrainValTest): @@ -60,6 +61,10 @@ class Model(object): self.loss = loss self.batch_size = batch_size + self.prep_d = None + self.seq = None + self.opt = None + self.build() def build(self): diff --git a/tests/icosagon/test_model.py b/tests/icosagon/test_model.py new file mode 100644 index 0000000..666960c --- /dev/null +++ b/tests/icosagon/test_model.py @@ -0,0 +1,35 @@ +from icosagon.data import Data +from icosagon.model import Model +from icosagon.trainprep import PreparedData +import torch +import ast + + +def _is_identity_function(f): + for x in range(-100, 101): + if f(x) != x: + return False + return True + + +def test_model_01(): + d = Data() + d.add_node_type('Dummy', 10) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Rel', torch.rand(10, 10).round()) + + m = Model(d) + + assert m.data == d + assert m.layer_dimensions == [32, 64] + assert (m.ratios.train, m.ratios.val, m.ratios.test) == (.8, .1, .1) + assert m.keep_prob == 1. + assert _is_identity_function(m.rel_activation) + assert m.layer_activation == torch.nn.functional.relu + assert _is_identity_function(m.dec_activation) + assert m.lr == 0.001 + assert m.loss == torch.nn.functional.binary_cross_entropy_with_logits + assert m.batch_size == 100 + assert isinstance(m.prep_d, PreparedData) + assert isinstance(m.seq, torch.nn.Sequential) + assert isinstance(m.opt, torch.optim.Optimizer) -- 2.26.2 From 90d40dbdbe25d02a69e37ab6691d1f909c461979 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 17 Jun 2020 12:03:44 +0200 Subject: [PATCH 106/227] Add test_model_02(). --- tests/icosagon/test_model.py | 55 ++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/tests/icosagon/test_model.py b/tests/icosagon/test_model.py index 666960c..c645d32 100644 --- a/tests/icosagon/test_model.py +++ b/tests/icosagon/test_model.py @@ -1,8 +1,15 @@ -from icosagon.data import Data +from icosagon.data import Data, \ + _equal from icosagon.model import Model -from icosagon.trainprep import PreparedData +from icosagon.trainprep import PreparedData, \ + PreparedRelationFamily, \ + PreparedRelationType, \ + TrainValTest, \ + norm_adj_mat_one_node_type import torch -import ast +from icosagon.input import OneHotInputLayer +from icosagon.convlayer import DecagonLayer +from icosagon.declayer import DecodeLayer def _is_identity_function(f): @@ -33,3 +40,45 @@ def test_model_01(): assert isinstance(m.prep_d, PreparedData) assert isinstance(m.seq, torch.nn.Sequential) assert isinstance(m.opt, torch.optim.Optimizer) + + +def test_model_02(): + d = Data() + d.add_node_type('Dummy', 10) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + mat = torch.rand(10, 10).round().to_sparse() + fam.add_relation_type('Dummy Rel', mat) + + m = Model(d, ratios=TrainValTest(1., 0., 0.)) + + assert isinstance(m.prep_d, PreparedData) + assert isinstance(m.prep_d.relation_families, list) + assert len(m.prep_d.relation_families) == 1 + assert isinstance(m.prep_d.relation_families[0], PreparedRelationFamily) + assert len(m.prep_d.relation_families[0].relation_types) == 1 + assert isinstance(m.prep_d.relation_families[0].relation_types[0], PreparedRelationType) + assert m.prep_d.relation_families[0].relation_types[0].adjacency_matrix_backward is None + assert torch.all(_equal(m.prep_d.relation_families[0].relation_types[0].adjacency_matrix, + norm_adj_mat_one_node_type(mat))) + + assert isinstance(m.seq[0], OneHotInputLayer) + assert isinstance(m.seq[1], DecagonLayer) + assert isinstance(m.seq[2], DecagonLayer) + assert isinstance(m.seq[3], DecodeLayer) + assert len(m.seq) == 4 + + +def test_model_03(): + d = Data() + d.add_node_type('Dummy', 10) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + mat = torch.rand(10, 10).round().to_sparse() + fam.add_relation_type('Dummy Rel', mat) + + m = Model(d, ratios=TrainValTest(1., 0., 0.)) + + state_dict = m.opt.state_dict() + assert isinstance(state_dict, dict) + # print(state_dict['param_groups']) + # print(list(m.seq.parameters())) + print(list(m.seq[1].parameters())) -- 2.26.2 From 0578b13ea4c04aa8b93e0fc31d4a89293f8c7b9d Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 17 Jun 2020 12:14:05 +0200 Subject: [PATCH 107/227] Fix prepare_adj_mat() by specifying shape explicitly. --- src/icosagon/trainprep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/icosagon/trainprep.py b/src/icosagon/trainprep.py index 3457791..6e8211b 100644 --- a/src/icosagon/trainprep.py +++ b/src/icosagon/trainprep.py @@ -111,7 +111,7 @@ def prepare_adj_mat(adj_mat: torch.Tensor, edges_neg = train_val_test_split_edges(edges_neg, ratios) adj_mat_train = torch.sparse_coo_tensor(indices = edges_pos.train.transpose(0, 1), - values=torch.ones(len(edges_pos.train), dtype=adj_mat.dtype)) + values=torch.ones(len(edges_pos.train), shape=adj_mat.shape, dtype=adj_mat.dtype)) return adj_mat_train, edges_pos, edges_neg -- 2.26.2 From cfb3d308a0346c3cb75f8d893a877bb17d3c8a87 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 17 Jun 2020 12:28:46 +0200 Subject: [PATCH 108/227] Add test_model_03(). --- src/icosagon/convolve.py | 2 +- src/icosagon/declayer.py | 2 +- src/icosagon/trainprep.py | 2 +- tests/icosagon/test_model.py | 6 +++++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/icosagon/convolve.py b/src/icosagon/convolve.py index 118a58e..364f61a 100644 --- a/src/icosagon/convolve.py +++ b/src/icosagon/convolve.py @@ -16,7 +16,7 @@ class GraphConv(torch.nn.Module): super().__init__(**kwargs) self.in_channels = in_channels self.out_channels = out_channels - self.weight = init_glorot(in_channels, out_channels) + self.weight = torch.nn.Parameter(init_glorot(in_channels, out_channels)) self.adjacency_matrix = adjacency_matrix diff --git a/src/icosagon/declayer.py b/src/icosagon/declayer.py index db78f11..8f6bff4 100644 --- a/src/icosagon/declayer.py +++ b/src/icosagon/declayer.py @@ -68,7 +68,7 @@ class DecodeLayer(torch.nn.Module): self.build() def build(self) -> None: - self.decoders = [] + self.decoders = torch.nn.ModuleList() for fam in self.data.relation_families: dec = fam.decoder_class(self.input_dim, len(fam.relation_types), self.keep_prob, self.activation) diff --git a/src/icosagon/trainprep.py b/src/icosagon/trainprep.py index 6e8211b..b7074e1 100644 --- a/src/icosagon/trainprep.py +++ b/src/icosagon/trainprep.py @@ -111,7 +111,7 @@ def prepare_adj_mat(adj_mat: torch.Tensor, edges_neg = train_val_test_split_edges(edges_neg, ratios) adj_mat_train = torch.sparse_coo_tensor(indices = edges_pos.train.transpose(0, 1), - values=torch.ones(len(edges_pos.train), shape=adj_mat.shape, dtype=adj_mat.dtype)) + values=torch.ones(len(edges_pos.train)), size=adj_mat.shape, dtype=adj_mat.dtype) return adj_mat_train, edges_pos, edges_neg diff --git a/tests/icosagon/test_model.py b/tests/icosagon/test_model.py index c645d32..5ee16ef 100644 --- a/tests/icosagon/test_model.py +++ b/tests/icosagon/test_model.py @@ -81,4 +81,8 @@ def test_model_03(): assert isinstance(state_dict, dict) # print(state_dict['param_groups']) # print(list(m.seq.parameters())) - print(list(m.seq[1].parameters())) + assert len(list(m.seq[0].parameters())) == 1 + assert len(list(m.seq[1].parameters())) == 1 + assert len(list(m.seq[2].parameters())) == 1 + assert len(list(m.seq[3].parameters())) == 2 + # print(list(m.seq[1].parameters())) -- 2.26.2 From 692f221f82dc8f4034c8bf37c211aea9d08beae8 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 17 Jun 2020 12:37:28 +0200 Subject: [PATCH 109/227] Add test_model_04(). --- tests/icosagon/test_model.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/icosagon/test_model.py b/tests/icosagon/test_model.py index 5ee16ef..c1ad0b6 100644 --- a/tests/icosagon/test_model.py +++ b/tests/icosagon/test_model.py @@ -86,3 +86,19 @@ def test_model_03(): assert len(list(m.seq[2].parameters())) == 1 assert len(list(m.seq[3].parameters())) == 2 # print(list(m.seq[1].parameters())) + + +def test_model_04(): + d = Data() + d.add_node_type('Dummy', 10) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + mat = torch.rand(10, 10).round().to_sparse() + fam.add_relation_type('Dummy Rel 1', mat) + fam.add_relation_type('Dummy Rel 2', mat.clone()) + + m = Model(d, ratios=TrainValTest(1., 0., 0.)) + + assert len(list(m.seq[0].parameters())) == 1 + assert len(list(m.seq[1].parameters())) == 2 + assert len(list(m.seq[2].parameters())) == 2 + assert len(list(m.seq[3].parameters())) == 3 -- 2.26.2 From aa541e0b42d9f27300336d60171c083f91d3cf4b Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 17 Jun 2020 12:43:04 +0200 Subject: [PATCH 110/227] Fix test_convolve. --- tests/icosagon/test_convolve.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/icosagon/test_convolve.py b/tests/icosagon/test_convolve.py index a916a89..5802800 100644 --- a/tests/icosagon/test_convolve.py +++ b/tests/icosagon/test_convolve.py @@ -13,7 +13,7 @@ def _test_graph_conv_01(use_sparse: bool): graph_conv = GraphConv(20, 20, adj_mat.to_sparse() \ if use_sparse else adj_mat) - graph_conv.weight = torch.eye(20) + graph_conv.weight = torch.nn.Parameter(torch.eye(20)) res = graph_conv(node_reprs) assert torch.all(res == adj_mat) @@ -28,7 +28,7 @@ def _test_graph_conv_02(use_sparse: bool): graph_conv = GraphConv(20, 20, adj_mat.to_sparse() \ if use_sparse else adj_mat) - graph_conv.weight = torch.eye(20) * 2 + graph_conv.weight = torch.nn.Parameter(torch.eye(20) * 2) res = graph_conv(node_reprs) assert torch.all(res == adj_mat * 2) @@ -57,14 +57,14 @@ def _test_graph_conv_03(use_sparse: bool): graph_conv = GraphConv(6, 3, adj_mat.to_sparse() \ if use_sparse else adj_mat) - graph_conv.weight = torch.tensor([ + graph_conv.weight = torch.nn.Parameter(torch.tensor([ [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1] - ], dtype=torch.float32) + ], dtype=torch.float32)) res = graph_conv(node_reprs) assert torch.all(res == expect) -- 2.26.2 From f6e1024428066aa276cd6b9ba11e563a2f0f39ca Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 17 Jun 2020 15:04:19 +0200 Subject: [PATCH 111/227] Add test_model_05(). --- tests/icosagon/test_model.py | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/icosagon/test_model.py b/tests/icosagon/test_model.py index c1ad0b6..2d39163 100644 --- a/tests/icosagon/test_model.py +++ b/tests/icosagon/test_model.py @@ -102,3 +102,48 @@ def test_model_04(): assert len(list(m.seq[1].parameters())) == 2 assert len(list(m.seq[2].parameters())) == 2 assert len(list(m.seq[3].parameters())) == 3 + + +def test_model_05(): + d = Data() + d.add_node_type('Dummy', 10) + + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + mat = torch.rand(10, 10).round().to_sparse() + fam.add_relation_type('Dummy Rel 1', mat) + fam.add_relation_type('Dummy Rel 2', mat.clone()) + + fam = d.add_relation_family('Dummy-Dummy 2', 0, 0, False) + mat = torch.rand(10, 10).round().to_sparse() + fam.add_relation_type('Dummy Rel 2-1', mat) + fam.add_relation_type('Dummy Rel 2-2', mat.clone()) + + m = Model(d, ratios=TrainValTest(1., 0., 0.)) + + assert len(list(m.seq[0].parameters())) == 1 + assert len(list(m.seq[1].parameters())) == 4 + assert len(list(m.seq[2].parameters())) == 4 + assert len(list(m.seq[3].parameters())) == 6 + + +def test_model_05(): + d = Data() + d.add_node_type('Dummy', 10) + d.add_node_type('Foobar', 20) + + fam = d.add_relation_family('Dummy-Foobar', 0, 1, True) + mat = torch.rand(10, 20).round().to_sparse() + fam.add_relation_type('Dummy Rel 1', mat) + fam.add_relation_type('Dummy Rel 2', mat.clone()) + + fam = d.add_relation_family('Dummy-Dummy 2', 0, 0, False) + mat = torch.rand(10, 10).round().to_sparse() + fam.add_relation_type('Dummy Rel 2-1', mat) + fam.add_relation_type('Dummy Rel 2-2', mat.clone()) + + m = Model(d, ratios=TrainValTest(1., 0., 0.)) + + assert len(list(m.seq[0].parameters())) == 2 + assert len(list(m.seq[1].parameters())) == 6 + assert len(list(m.seq[2].parameters())) == 6 + assert len(list(m.seq[3].parameters())) == 6 -- 2.26.2 From 414c93419fe1f36e481641a76377a7c56fb0d0f6 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 17 Jun 2020 15:13:56 +0200 Subject: [PATCH 112/227] Add test_model_06(). --- src/icosagon/data.py | 4 ++++ tests/icosagon/test_model.py | 38 ++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/src/icosagon/data.py b/src/icosagon/data.py index 0181b7d..7e39db7 100644 --- a/src/icosagon/data.py +++ b/src/icosagon/data.py @@ -120,6 +120,10 @@ class RelationFamily(RelationFamilyBase): adjacency_matrix.transpose(0, 1))): raise ValueError('Relation family is symmetric but adjacency_matrix is assymetric') + if not self.is_symmetric and node_type_row != node_type_column and \ + adjacency_matrix_backward is None: + raise ValueError('Relation is asymmetric but adjacency_matrix_backward is None') + if self.is_symmetric and node_type_row != node_type_column: adjacency_matrix_backward = adjacency_matrix.transpose(0, 1) diff --git a/tests/icosagon/test_model.py b/tests/icosagon/test_model.py index 2d39163..3084031 100644 --- a/tests/icosagon/test_model.py +++ b/tests/icosagon/test_model.py @@ -10,6 +10,7 @@ import torch from icosagon.input import OneHotInputLayer from icosagon.convlayer import DecagonLayer from icosagon.declayer import DecodeLayer +import pytest def _is_identity_function(f): @@ -147,3 +148,40 @@ def test_model_05(): assert len(list(m.seq[1].parameters())) == 6 assert len(list(m.seq[2].parameters())) == 6 assert len(list(m.seq[3].parameters())) == 6 + + +def test_model_06(): + d = Data() + d.add_node_type('Dummy', 10) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Rel', torch.rand(10, 10).round()) + + with pytest.raises(TypeError): + m = Model(1) + + with pytest.raises(TypeError): + m = Model(d, layer_dimensions=1) + + with pytest.raises(TypeError): + m = Model(d, ratios=1) + + with pytest.raises(ValueError): + m = Model(d, keep_prob='x') + + with pytest.raises(TypeError): + m = Model(d, rel_activation='x') + + with pytest.raises(TypeError): + m = Model(d, layer_activation='x') + + with pytest.raises(TypeError): + m = Model(d, dec_activation='x') + + with pytest.raises(ValueError): + m = Model(d, lr='x') + + with pytest.raises(TypeError): + m = Model(d, loss=1) + + with pytest.raises(ValueError): + m = Model(d, batch_size='x') -- 2.26.2 From 190a00e6e53e02efbd54f003315ec8ede13730a9 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 17 Jun 2020 15:17:08 +0200 Subject: [PATCH 113/227] Fix test_input. --- tests/icosagon/test_input.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/icosagon/test_input.py b/tests/icosagon/test_input.py index 4f50912..8bb5dc0 100644 --- a/tests/icosagon/test_input.py +++ b/tests/icosagon/test_input.py @@ -10,7 +10,7 @@ def _some_data(): d.add_node_type('Gene', 1000) d.add_node_type('Drug', 100) - fam = d.add_relation_family('Drug-Gene', 1, 0, False) + fam = d.add_relation_family('Drug-Gene', 1, 0, True) fam.add_relation_type('Target', torch.rand(100, 1000)) fam = d.add_relation_family('Gene-Gene', 0, 0, False) -- 2.26.2 From f17f97caf4182231f97e40f32ac0039acff29f7b Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 17 Jun 2020 15:33:00 +0200 Subject: [PATCH 114/227] Add test_model_07(). --- src/icosagon/model.py | 8 ++++---- tests/icosagon/test_model.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/icosagon/model.py b/src/icosagon/model.py index 5d384f0..321e600 100644 --- a/src/icosagon/model.py +++ b/src/icosagon/model.py @@ -99,13 +99,13 @@ class Model(object): def run_epoch(self): pred = self.seq(None) - batch = PredictionsBatch(pred, self.batch_size) + batch = PredictionsBatch(pred, batch_size=self.batch_size) n = len(list(iter(batch))) loss_sum = 0 - for i in range(n - 1): + for i in range(n): self.opt.zero_grad() pred = self.seq(None) - batch = PredictionsBatch(pred, self.batch_size, shuffle=True) + batch = PredictionsBatch(pred, batch_size=self.batch_size, shuffle=True) seed = torch.rand(1).item() rng_state = torch.get_rng_state() torch.manual_seed(seed) @@ -116,7 +116,7 @@ class Model(object): (input, target) = next(it) loss = self.loss(input, target) loss.backward() - self.opt.optimize() + self.opt.step() loss_sum += loss.detach().cpu().item() return loss_sum diff --git a/tests/icosagon/test_model.py b/tests/icosagon/test_model.py index 3084031..fce7460 100644 --- a/tests/icosagon/test_model.py +++ b/tests/icosagon/test_model.py @@ -185,3 +185,14 @@ def test_model_06(): with pytest.raises(ValueError): m = Model(d, batch_size='x') + + +def test_model_07(): + d = Data() + d.add_node_type('Dummy', 10) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Rel', torch.rand(10, 10).round()) + + m = Model(d) + + m.run_epoch() -- 2.26.2 From f11f25704a54de1f1f1918fb54b6bcfdeef2384f Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 18 Jun 2020 09:40:36 +0200 Subject: [PATCH 115/227] Make Model accept PreparedData rather than Data. --- src/icosagon/model.py | 19 +++-------- tests/icosagon/test_model.py | 61 ++++++++++++++++++++++-------------- 2 files changed, 43 insertions(+), 37 deletions(-) diff --git a/src/icosagon/model.py b/src/icosagon/model.py index 321e600..7261129 100644 --- a/src/icosagon/model.py +++ b/src/icosagon/model.py @@ -1,8 +1,7 @@ from .data import Data from typing import List, \ Callable -from .trainprep import prepare_training, \ - TrainValTest +from .trainprep import PreparedData import torch from .convlayer import DecagonLayer from .input import OneHotInputLayer @@ -12,9 +11,8 @@ from .batch import PredictionsBatch class Model(object): - def __init__(self, data: Data, + def __init__(self, prep_d: PreparedData, layer_dimensions: List[int] = [32, 64], - ratios: TrainValTest = TrainValTest(.8, .1, .1), keep_prob: float = 1., rel_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, layer_activation: Callable[[torch.Tensor], torch.Tensor] = torch.nn.functional.relu, @@ -23,15 +21,12 @@ class Model(object): loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = torch.nn.functional.binary_cross_entropy_with_logits, batch_size: int = 100) -> None: - if not isinstance(data, Data): - raise TypeError('data must be an instance of Data') + if not isinstance(prep_d, PreparedData): + raise TypeError('prep_d must be an instance of PreparedData') if not isinstance(layer_dimensions, list): raise TypeError('layer_dimensions must be a list') - if not isinstance(ratios, TrainValTest): - raise TypeError('ratios must be an instance of TrainValTest') - keep_prob = float(keep_prob) if not isinstance(rel_activation, FunctionType): @@ -50,9 +45,8 @@ class Model(object): batch_size = int(batch_size) - self.data = data + self.prep_d = prep_d self.layer_dimensions = layer_dimensions - self.ratios = ratios self.keep_prob = keep_prob self.rel_activation = rel_activation self.layer_activation = layer_activation @@ -61,15 +55,12 @@ class Model(object): self.loss = loss self.batch_size = batch_size - self.prep_d = None self.seq = None self.opt = None self.build() def build(self): - self.prep_d = prepare_training(self.data, self.ratios) - in_layer = OneHotInputLayer(self.prep_d) last_output_dim = in_layer.output_dim seq = [ in_layer ] diff --git a/tests/icosagon/test_model.py b/tests/icosagon/test_model.py index fce7460..960e34d 100644 --- a/tests/icosagon/test_model.py +++ b/tests/icosagon/test_model.py @@ -5,7 +5,8 @@ from icosagon.trainprep import PreparedData, \ PreparedRelationFamily, \ PreparedRelationType, \ TrainValTest, \ - norm_adj_mat_one_node_type + norm_adj_mat_one_node_type, \ + prepare_training import torch from icosagon.input import OneHotInputLayer from icosagon.convlayer import DecagonLayer @@ -26,11 +27,12 @@ def test_model_01(): fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) fam.add_relation_type('Dummy Rel', torch.rand(10, 10).round()) - m = Model(d) + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) - assert m.data == d + m = Model(prep_d) + + assert m.prep_d == prep_d assert m.layer_dimensions == [32, 64] - assert (m.ratios.train, m.ratios.val, m.ratios.test) == (.8, .1, .1) assert m.keep_prob == 1. assert _is_identity_function(m.rel_activation) assert m.layer_activation == torch.nn.functional.relu @@ -38,7 +40,6 @@ def test_model_01(): assert m.lr == 0.001 assert m.loss == torch.nn.functional.binary_cross_entropy_with_logits assert m.batch_size == 100 - assert isinstance(m.prep_d, PreparedData) assert isinstance(m.seq, torch.nn.Sequential) assert isinstance(m.opt, torch.optim.Optimizer) @@ -50,7 +51,9 @@ def test_model_02(): mat = torch.rand(10, 10).round().to_sparse() fam.add_relation_type('Dummy Rel', mat) - m = Model(d, ratios=TrainValTest(1., 0., 0.)) + prep_d = prepare_training(d, TrainValTest(1., 0., 0.)) + + m = Model(prep_d) assert isinstance(m.prep_d, PreparedData) assert isinstance(m.prep_d.relation_families, list) @@ -76,7 +79,9 @@ def test_model_03(): mat = torch.rand(10, 10).round().to_sparse() fam.add_relation_type('Dummy Rel', mat) - m = Model(d, ratios=TrainValTest(1., 0., 0.)) + prep_d = prepare_training(d, TrainValTest(1., 0., 0.)) + + m = Model(prep_d) state_dict = m.opt.state_dict() assert isinstance(state_dict, dict) @@ -97,7 +102,9 @@ def test_model_04(): fam.add_relation_type('Dummy Rel 1', mat) fam.add_relation_type('Dummy Rel 2', mat.clone()) - m = Model(d, ratios=TrainValTest(1., 0., 0.)) + prep_d = prepare_training(d, TrainValTest(1., 0., 0.)) + + m = Model(prep_d) assert len(list(m.seq[0].parameters())) == 1 assert len(list(m.seq[1].parameters())) == 2 @@ -119,7 +126,9 @@ def test_model_05(): fam.add_relation_type('Dummy Rel 2-1', mat) fam.add_relation_type('Dummy Rel 2-2', mat.clone()) - m = Model(d, ratios=TrainValTest(1., 0., 0.)) + prep_d = prepare_training(d, TrainValTest(1., 0., 0.)) + + m = Model(prep_d) assert len(list(m.seq[0].parameters())) == 1 assert len(list(m.seq[1].parameters())) == 4 @@ -127,7 +136,7 @@ def test_model_05(): assert len(list(m.seq[3].parameters())) == 6 -def test_model_05(): +def test_model_06(): d = Data() d.add_node_type('Dummy', 10) d.add_node_type('Foobar', 20) @@ -142,7 +151,9 @@ def test_model_05(): fam.add_relation_type('Dummy Rel 2-1', mat) fam.add_relation_type('Dummy Rel 2-2', mat.clone()) - m = Model(d, ratios=TrainValTest(1., 0., 0.)) + prep_d = prepare_training(d, TrainValTest(1., 0., 0.)) + + m = Model(prep_d) assert len(list(m.seq[0].parameters())) == 2 assert len(list(m.seq[1].parameters())) == 6 @@ -150,49 +161,53 @@ def test_model_05(): assert len(list(m.seq[3].parameters())) == 6 -def test_model_06(): +def test_model_07(): d = Data() d.add_node_type('Dummy', 10) fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) fam.add_relation_type('Dummy Rel', torch.rand(10, 10).round()) + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) + with pytest.raises(TypeError): m = Model(1) with pytest.raises(TypeError): - m = Model(d, layer_dimensions=1) + m = Model(prep_d, layer_dimensions=1) with pytest.raises(TypeError): - m = Model(d, ratios=1) + m = Model(prep_d, ratios=1) with pytest.raises(ValueError): - m = Model(d, keep_prob='x') + m = Model(prep_d, keep_prob='x') with pytest.raises(TypeError): - m = Model(d, rel_activation='x') + m = Model(prep_d, rel_activation='x') with pytest.raises(TypeError): - m = Model(d, layer_activation='x') + m = Model(prep_d, layer_activation='x') with pytest.raises(TypeError): - m = Model(d, dec_activation='x') + m = Model(prep_d, dec_activation='x') with pytest.raises(ValueError): - m = Model(d, lr='x') + m = Model(prep_d, lr='x') with pytest.raises(TypeError): - m = Model(d, loss=1) + m = Model(prep_d, loss=1) with pytest.raises(ValueError): - m = Model(d, batch_size='x') + m = Model(prep_d, batch_size='x') -def test_model_07(): +def test_model_08(): d = Data() d.add_node_type('Dummy', 10) fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) fam.add_relation_type('Dummy Rel', torch.rand(10, 10).round()) - m = Model(d) + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) + + m = Model(prep_d) m.run_epoch() -- 2.26.2 From 9366687239d7b6b4fd296624045a7633dff4a691 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 18 Jun 2020 13:00:17 +0200 Subject: [PATCH 116/227] Add TrainLoop. --- src/icosagon/model.py | 58 +++------------------------ src/icosagon/trainloop.py | 69 ++++++++++++++++++++++++++++++++ tests/icosagon/test_trainloop.py | 24 +++++++++++ 3 files changed, 99 insertions(+), 52 deletions(-) create mode 100644 src/icosagon/trainloop.py create mode 100644 tests/icosagon/test_trainloop.py diff --git a/src/icosagon/model.py b/src/icosagon/model.py index 7261129..1c9e413 100644 --- a/src/icosagon/model.py +++ b/src/icosagon/model.py @@ -10,16 +10,16 @@ from .declayer import DecodeLayer from .batch import PredictionsBatch -class Model(object): +class Model(torch.nn.Module): def __init__(self, prep_d: PreparedData, layer_dimensions: List[int] = [32, 64], keep_prob: float = 1., rel_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, layer_activation: Callable[[torch.Tensor], torch.Tensor] = torch.nn.functional.relu, dec_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, - lr: float = 0.001, - loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = torch.nn.functional.binary_cross_entropy_with_logits, - batch_size: int = 100) -> None: + **kwargs) -> None: + + super().__init__(**kwargs) if not isinstance(prep_d, PreparedData): raise TypeError('prep_d must be an instance of PreparedData') @@ -38,25 +38,14 @@ class Model(object): if not isinstance(dec_activation, FunctionType): raise TypeError('dec_activation must be a function') - lr = float(lr) - - if not isinstance(loss, FunctionType): - raise TypeError('loss must be a function') - - batch_size = int(batch_size) - self.prep_d = prep_d self.layer_dimensions = layer_dimensions self.keep_prob = keep_prob self.rel_activation = rel_activation self.layer_activation = layer_activation self.dec_activation = dec_activation - self.lr = lr - self.loss = loss - self.batch_size = batch_size self.seq = None - self.opt = None self.build() @@ -84,40 +73,5 @@ class Model(object): seq = torch.nn.Sequential(*seq) self.seq = seq - opt = torch.optim.Adam(seq.parameters(), lr=self.lr) - self.opt = opt - - - def run_epoch(self): - pred = self.seq(None) - batch = PredictionsBatch(pred, batch_size=self.batch_size) - n = len(list(iter(batch))) - loss_sum = 0 - for i in range(n): - self.opt.zero_grad() - pred = self.seq(None) - batch = PredictionsBatch(pred, batch_size=self.batch_size, shuffle=True) - seed = torch.rand(1).item() - rng_state = torch.get_rng_state() - torch.manual_seed(seed) - it = iter(batch) - torch.set_rng_state(rng_state) - for k in range(i): - _ = next(it) - (input, target) = next(it) - loss = self.loss(input, target) - loss.backward() - self.opt.step() - loss_sum += loss.detach().cpu().item() - return loss_sum - - - def train(self, max_epochs): - best_loss = None - best_epoch = None - for i in range(max_epochs): - loss = self.run_epoch() - if best_loss is None or loss < best_loss: - best_loss = loss - best_epoch = i - return loss, best_loss, best_epoch + def forward(self, _): + return self.seq(None) diff --git a/src/icosagon/trainloop.py b/src/icosagon/trainloop.py new file mode 100644 index 0000000..051019e --- /dev/null +++ b/src/icosagon/trainloop.py @@ -0,0 +1,69 @@ +from .model import Model +import torch +from .batch import PredictionsBatch +from typing import Callable +from types import FunctionType + + +class TrainLoop(object): + def __init__(self, model: Model, lr: float = 0.001, + loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = \ + torch.nn.functional.binary_cross_entropy_with_logits, + batch_size: int = 100) -> None: + + if not isinstance(model, Model): + raise TypeError('model must be an instance of Model') + + lr = float(lr) + + if not isinstance(loss, FunctionType): + raise TypeError('loss must be a function') + + batch_size = int(batch_size) + + self.model = model + self.lr = lr + self.loss = loss + self.batch_size = batch_size + + self.opt = None + + self.build() + + def build(self) -> None: + opt = torch.optim.Adam(self.model.parameters(), lr=self.lr) + self.opt = opt + + def run_epoch(self): + pred = self.model(None) + batch = PredictionsBatch(pred, batch_size=self.batch_size) + n = len(list(iter(batch))) + loss_sum = 0 + for i in range(n): + self.opt.zero_grad() + pred = self.seq(None) + batch = PredictionsBatch(pred, batch_size=self.batch_size, shuffle=True) + seed = torch.rand(1).item() + rng_state = torch.get_rng_state() + torch.manual_seed(seed) + it = iter(batch) + torch.set_rng_state(rng_state) + for k in range(i): + _ = next(it) + (input, target) = next(it) + loss = self.loss(input, target) + loss.backward() + self.opt.step() + loss_sum += loss.detach().cpu().item() + return loss_sum + + + def train(self, max_epochs): + best_loss = None + best_epoch = None + for i in range(max_epochs): + loss = self.run_epoch() + if best_loss is None or loss < best_loss: + best_loss = loss + best_epoch = i + return loss, best_loss, best_epoch diff --git a/tests/icosagon/test_trainloop.py b/tests/icosagon/test_trainloop.py new file mode 100644 index 0000000..2476c9c --- /dev/null +++ b/tests/icosagon/test_trainloop.py @@ -0,0 +1,24 @@ +from icosagon.data import Data +from icosagon.trainprep import prepare_training, \ + TrainValTest +from icosagon.model import Model +from icosagon.trainloop import TrainLoop +import torch + + +def test_train_loop_01(): + d = Data() + d.add_node_type('Dummy', 10) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Rel', torch.rand(10, 10).round()) + + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) + + m = Model(prep_d) + + loop = TrainLoop(m) + + assert loop.model == m + assert loop.lr == 0.001 + assert loop.loss == torch.nn.functional.binary_cross_entropy_with_logits + assert loop.batch_size == 100 -- 2.26.2 From 2b388e4431c4a23ef879f1c7d4dea0943b7f0ac7 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 18 Jun 2020 13:31:18 +0200 Subject: [PATCH 117/227] Fix test_model. --- src/icosagon/trainloop.py | 2 +- tests/icosagon/test_model.py | 31 ------------------------------- tests/icosagon/test_trainloop.py | 15 +++++++++++++++ 3 files changed, 16 insertions(+), 32 deletions(-) diff --git a/src/icosagon/trainloop.py b/src/icosagon/trainloop.py index 051019e..2ce8240 100644 --- a/src/icosagon/trainloop.py +++ b/src/icosagon/trainloop.py @@ -41,7 +41,7 @@ class TrainLoop(object): loss_sum = 0 for i in range(n): self.opt.zero_grad() - pred = self.seq(None) + pred = self.model(None) batch = PredictionsBatch(pred, batch_size=self.batch_size, shuffle=True) seed = torch.rand(1).item() rng_state = torch.get_rng_state() diff --git a/tests/icosagon/test_model.py b/tests/icosagon/test_model.py index 960e34d..5c122d9 100644 --- a/tests/icosagon/test_model.py +++ b/tests/icosagon/test_model.py @@ -37,11 +37,7 @@ def test_model_01(): assert _is_identity_function(m.rel_activation) assert m.layer_activation == torch.nn.functional.relu assert _is_identity_function(m.dec_activation) - assert m.lr == 0.001 - assert m.loss == torch.nn.functional.binary_cross_entropy_with_logits - assert m.batch_size == 100 assert isinstance(m.seq, torch.nn.Sequential) - assert isinstance(m.opt, torch.optim.Optimizer) def test_model_02(): @@ -83,15 +79,10 @@ def test_model_03(): m = Model(prep_d) - state_dict = m.opt.state_dict() - assert isinstance(state_dict, dict) - # print(state_dict['param_groups']) - # print(list(m.seq.parameters())) assert len(list(m.seq[0].parameters())) == 1 assert len(list(m.seq[1].parameters())) == 1 assert len(list(m.seq[2].parameters())) == 1 assert len(list(m.seq[3].parameters())) == 2 - # print(list(m.seq[1].parameters())) def test_model_04(): @@ -189,25 +180,3 @@ def test_model_07(): with pytest.raises(TypeError): m = Model(prep_d, dec_activation='x') - - with pytest.raises(ValueError): - m = Model(prep_d, lr='x') - - with pytest.raises(TypeError): - m = Model(prep_d, loss=1) - - with pytest.raises(ValueError): - m = Model(prep_d, batch_size='x') - - -def test_model_08(): - d = Data() - d.add_node_type('Dummy', 10) - fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) - fam.add_relation_type('Dummy Rel', torch.rand(10, 10).round()) - - prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) - - m = Model(prep_d) - - m.run_epoch() diff --git a/tests/icosagon/test_trainloop.py b/tests/icosagon/test_trainloop.py index 2476c9c..04956c6 100644 --- a/tests/icosagon/test_trainloop.py +++ b/tests/icosagon/test_trainloop.py @@ -22,3 +22,18 @@ def test_train_loop_01(): assert loop.lr == 0.001 assert loop.loss == torch.nn.functional.binary_cross_entropy_with_logits assert loop.batch_size == 100 + + +def test_train_loop_02(): + d = Data() + d.add_node_type('Dummy', 10) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Rel', torch.rand(10, 10).round()) + + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) + + m = Model(prep_d) + + loop = TrainLoop(m) + + loop.run_epoch() -- 2.26.2 From c023d35c4676d675219d7fa9f5152d8d2b92f970 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 18 Jun 2020 18:27:02 +0200 Subject: [PATCH 118/227] Introduce FlatPredictions, flatten_predictions(), BatchIndices, gather_batch_indices(). --- src/icosagon/batch.py | 107 +++++++++++++++++++++++-------- src/icosagon/trainloop.py | 39 +++++++----- tests/icosagon/test_batch.py | 120 ++++++++++++++++++++++++++++++++++- 3 files changed, 222 insertions(+), 44 deletions(-) diff --git a/src/icosagon/batch.py b/src/icosagon/batch.py index 9a28712..e395f3f 100644 --- a/src/icosagon/batch.py +++ b/src/icosagon/batch.py @@ -1,13 +1,72 @@ -from icosagon.declayer import Predictions +from .declayer import Predictions import torch +from dataclasses import dataclass +from .trainprep import PreparedData +from typing import Tuple + + +@dataclass +class FlatPredictions(object): + predictions: torch.Tensor + truth: torch.Tensor + part_type: str + + +def flatten_predictions(pred: Predictions, part_type: str = 'train'): + if not isinstance(pred, Predictions): + raise TypeError('pred must be an instance of Predictions') + + if part_type not in ['train', 'val', 'test']: + raise ValueError('part_type must be set to train, val or test') + + edge_types = [('edges_pos', 1), ('edges_neg', 0), + ('edges_back_pos', 1), ('edges_back_neg', 0)] + + input = [] + target = [] + + for fam in pred.relation_families: + for rel in fam.relation_types: + for (et, tgt) in edge_types: + edge_pred = getattr(getattr(rel, et), part_type) + input.append(edge_pred) + target.append(torch.ones_like(edge_pred) * tgt) + + input = torch.cat(input) + target = torch.cat(target) + + return FlatPredictions(input, target, part_type) + + +@dataclass +class BatchIndices(object): + indices: torch.Tensor + part_type: str + + +def gather_batch_indices(pred: FlatPredictions, + indices: BatchIndices) -> Tuple[torch.Tensor, torch.Tensor]: + + if not isinstance(pred, FlatPredictions): + raise TypeError('pred must be an instance of FlatPredictions') + + if not isinstance(indices, BatchIndices): + raise TypeError('indices must be an instance of BatchIndices') + + if pred.part_type != indices.part_type: + raise ValueError('part_type must be the same in pred and indices') + + return (pred.predictions[indices.indices], + pred.truth[indices.indices]) class PredictionsBatch(object): - def __init__(self, pred: Predictions, part_type: str = 'train', - batch_size: int = 100, shuffle: bool = False) -> None: + def __init__(self, prep_d: PreparedData, part_type: str = 'train', + batch_size: int = 100, shuffle: bool = False, + generator: torch.Generator = None) -> None: - if not isinstance(pred, Predictions): - raise TypeError('pred must be an instance of Predictions') + if not isinstance(prep_d, PreparedData): + raise TypeError('prep_d must be an instance of PreparedData') if part_type not in ['train', 'val', 'test']: raise ValueError('part_type must be set to train, val or test') @@ -16,32 +75,28 @@ class PredictionsBatch(object): shuffle = bool(shuffle) - self.predictions = pred + if generator is not None and not isinstance(generator, torch.Generator): + raise TypeError('generator must be an instance of torch.Generator') + + self.prep_d = prep_d self.part_type = part_type self.batch_size = batch_size self.shuffle = shuffle + self.generator = generator or torch.default_generator - def __iter__(self): - edge_types = [('edges_pos', 1), ('edges_neg', 0), - ('edges_back_pos', 1), ('edges_back_neg', 0)] - - input = [] - target = [] - - for fam in self.predictions.relation_families: + count = 0 + for fam in prep_d.relation_families: for rel in fam.relation_types: - for (et, tgt) in edge_types: - edge_pred = getattr(getattr(rel, et), self.part_type) - input.append(edge_pred) - target.append(torch.ones_like(edge_pred) * tgt) - - input = torch.cat(input) - target = torch.cat(target) + for et in ['edges_pos', 'edges_neg', + 'edges_back_pos', 'edges_back_neg']: + count += len(getattr(getattr(rel, et), part_type)) + self.total_edge_count = count + def __iter__(self): + values = torch.arange(self.total_edge_count) if self.shuffle: - perm = torch.randperm(len(input)) - input = input[perm] - target = target[perm] + perm = torch.randperm(len(values)) + values = values[perm] - for i in range(0, len(input), self.batch_size): - yield (input[i:i+self.batch_size], target[i:i+self.batch_size]) + for i in range(0, len(values), self.batch_size): + yield BatchIndices(values[i:i+self.batch_size], self.part_type) diff --git a/src/icosagon/trainloop.py b/src/icosagon/trainloop.py index 2ce8240..5492878 100644 --- a/src/icosagon/trainloop.py +++ b/src/icosagon/trainloop.py @@ -1,6 +1,8 @@ from .model import Model import torch -from .batch import PredictionsBatch +from .batch import PredictionsBatch, \ + flatten_predictions, \ + gather_batch_indices from typing import Callable from types import FunctionType @@ -9,7 +11,7 @@ class TrainLoop(object): def __init__(self, model: Model, lr: float = 0.001, loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = \ torch.nn.functional.binary_cross_entropy_with_logits, - batch_size: int = 100) -> None: + batch_size: int = 100, generator: torch.Generator = None) -> None: if not isinstance(model, Model): raise TypeError('model must be an instance of Model') @@ -21,10 +23,14 @@ class TrainLoop(object): batch_size = int(batch_size) + if generator is not None and not isinstance(generator, torch.Generator): + raise TypeError('generator must be an instance of torch.Generator') + self.model = model self.lr = lr self.loss = loss self.batch_size = batch_size + self.generator = generator or torch.default_generator self.opt = None @@ -35,22 +41,25 @@ class TrainLoop(object): self.opt = opt def run_epoch(self): - pred = self.model(None) - batch = PredictionsBatch(pred, batch_size=self.batch_size) - n = len(list(iter(batch))) + batch = PredictionsBatch(self.model.prep_d, batch_size=self.batch_size, + generator=self.generator) + # pred = self.model(None) + # n = len(list(iter(batch))) loss_sum = 0 - for i in range(n): + for indices in batch: self.opt.zero_grad() pred = self.model(None) - batch = PredictionsBatch(pred, batch_size=self.batch_size, shuffle=True) - seed = torch.rand(1).item() - rng_state = torch.get_rng_state() - torch.manual_seed(seed) - it = iter(batch) - torch.set_rng_state(rng_state) - for k in range(i): - _ = next(it) - (input, target) = next(it) + pred = flatten_predictions(pred) + # batch = PredictionsBatch(pred, batch_size=self.batch_size, shuffle=True) + # seed = torch.rand(1).item() + # rng_state = torch.get_rng_state() + # torch.manual_seed(seed) + #it = iter(batch) + #torch.set_rng_state(rng_state) + #for k in range(i): + #_ = next(it) + #(input, target) = next(it) + (input, target) = gather_batch_indices(pred, indices) loss = self.loss(input, target) loss.backward() self.opt.step() diff --git a/tests/icosagon/test_batch.py b/tests/icosagon/test_batch.py index 3d185e4..b6cd6d6 100644 --- a/tests/icosagon/test_batch.py +++ b/tests/icosagon/test_batch.py @@ -1,4 +1,8 @@ -from icosagon.batch import PredictionsBatch +from icosagon.batch import PredictionsBatch, \ + FlatPredictions, \ + flatten_predictions, \ + BatchIndices, \ + gather_batch_indices from icosagon.declayer import Predictions, \ RelationPredictions, \ RelationFamilyPredictions @@ -6,6 +10,113 @@ from icosagon.trainprep import prepare_training, \ TrainValTest from icosagon.data import Data import torch +import pytest + + +def test_flat_predictions_01(): + pred = FlatPredictions(torch.tensor([0, 1, 0, 1]), + torch.tensor([1, 0, 1, 0]), 'train') + + assert torch.all(pred.predictions == torch.tensor([0, 1, 0, 1])) + assert torch.all(pred.truth == torch.tensor([1, 0, 1, 0])) + assert pred.part_type == 'train' + + +def test_flatten_predictions_01(): + rel_pred = RelationPredictions( + TrainValTest(torch.tensor([1, 0, 1, 0, 1], dtype=torch.float32), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.tensor([1, 0, 1, 0, 1], dtype=torch.float32), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)) + ) + fam_pred = RelationFamilyPredictions([ rel_pred ]) + pred = Predictions([ fam_pred ]) + + pred_flat = flatten_predictions(pred, part_type='train') + + assert torch.all(pred_flat.predictions == \ + torch.tensor([1, 0, 1, 0, 1, 1, 0, 1, 0, 1], dtype=torch.float32)) + assert torch.all(pred_flat.truth == \ + torch.tensor([1, 1, 1, 1, 1, 0, 0, 0, 0, 0], dtype=torch.float32)) + assert pred_flat.part_type == 'train' + + +def test_flatten_predictions_02(): + rel_pred = RelationPredictions( + TrainValTest(torch.tensor([1, 0, 1, 0, 1], dtype=torch.float32), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.tensor([1, 0, 1, 0, 1], dtype=torch.float32), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)) + ) + fam_pred = RelationFamilyPredictions([ rel_pred ]) + pred = Predictions([ fam_pred ]) + + pred_flat = flatten_predictions(pred, part_type='val') + + assert len(pred_flat.predictions) == 0 + assert len(pred_flat.truth) == 0 + assert pred_flat.part_type == 'val' + + +def test_flatten_predictions_03(): + rel_pred = RelationPredictions( + TrainValTest(torch.tensor([1, 0, 1, 0, 1], dtype=torch.float32), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.tensor([1, 0, 1, 0, 1], dtype=torch.float32), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)) + ) + fam_pred = RelationFamilyPredictions([ rel_pred ]) + pred = Predictions([ fam_pred ]) + + pred_flat = flatten_predictions(pred, part_type='test') + + assert len(pred_flat.predictions) == 0 + assert len(pred_flat.truth) == 0 + assert pred_flat.part_type == 'test' + + +def test_flatten_predictions_04(): + rel_pred = RelationPredictions( + TrainValTest(torch.tensor([1, 0, 1, 0, 1], dtype=torch.float32), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.tensor([1, 0, 1, 0, 1], dtype=torch.float32), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)) + ) + fam_pred = RelationFamilyPredictions([ rel_pred ]) + pred = Predictions([ fam_pred ]) + + with pytest.raises(TypeError): + pred_flat = flatten_predictions(1, part_type='test') + + with pytest.raises(ValueError): + pred_flat = flatten_predictions(pred, part_type='x') + + +def test_batch_indices_01(): + indices = BatchIndices(torch.tensor([0, 1, 2, 3, 4]), 'train') + assert torch.all(indices.indices == torch.tensor([0, 1, 2, 3, 4])) + assert indices.part_type == 'train' + + +def test_gather_batch_indices_01(): + rel_pred = RelationPredictions( + TrainValTest(torch.tensor([1, 0, 1, 0, 1], dtype=torch.float32), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.tensor([1, 0, 1, 0, 1], dtype=torch.float32), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)) + ) + fam_pred = RelationFamilyPredictions([ rel_pred ]) + pred = Predictions([ fam_pred ]) + + pred_flat = flatten_predictions(pred, part_type='train') + + indices = BatchIndices(torch.tensor([0, 2, 4, 5, 7, 9]), 'train') + + (input, target) = gather_batch_indices(pred_flat, indices) + assert torch.all(input == \ + torch.tensor([1, 1, 1, 1, 1, 1], dtype=torch.float32)) + assert torch.all(target == \ + torch.tensor([1, 1, 1, 0, 0, 0], dtype=torch.float32)) def test_predictions_batch_01(): @@ -38,10 +149,13 @@ def test_predictions_batch_01(): fam_pred = RelationFamilyPredictions([ rel_pred ]) pred = Predictions([ fam_pred ]) - batch = PredictionsBatch(pred, part_type='train', batch_size=1) + pred_flat = flatten_predictions(pred, part_type='train') + + batch = PredictionsBatch(prep_d, part_type='train', batch_size=1) count = 0 lst = [] - for (input, target) in batch: + for indices in batch: + (input, target) = gather_batch_indices(pred_flat, indices) assert len(input) == 1 assert len(target) == 1 lst.append((input[0], target[0])) -- 2.26.2 From f6006dcc847d5fae8c4748a983263c2e49221a47 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 19 Jun 2020 20:14:14 +0200 Subject: [PATCH 119/227] Add test_model_08(). --- tests/icosagon/test_model.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/icosagon/test_model.py b/tests/icosagon/test_model.py index 5c122d9..5e5d482 100644 --- a/tests/icosagon/test_model.py +++ b/tests/icosagon/test_model.py @@ -180,3 +180,25 @@ def test_model_07(): with pytest.raises(TypeError): m = Model(prep_d, dec_activation='x') + + +def test_model_08(): + d = Data() + d.add_node_type('Dummy', 10) + d.add_node_type('Foobar', 20) + + fam = d.add_relation_family('Dummy-Foobar', 0, 1, True) + mat = torch.rand(10, 20).round().to_sparse() + fam.add_relation_type('Dummy Rel 1', mat) + fam.add_relation_type('Dummy Rel 2', mat.clone()) + + fam = d.add_relation_family('Dummy-Dummy 2', 0, 0, False) + mat = torch.rand(10, 10).round().to_sparse() + fam.add_relation_type('Dummy Rel 2-1', mat) + fam.add_relation_type('Dummy Rel 2-2', mat.clone()) + + prep_d = prepare_training(d, TrainValTest(1., 0., 0.)) + + m = Model(prep_d) + + assert len(list(m.parameters())) == 20 -- 2.26.2 From 03478e2667d3af8bbaeb991911f88bb8c16826f3 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 19 Jun 2020 21:00:08 +0200 Subject: [PATCH 120/227] Add test_parameter_count_01(). --- src/icosagon/convlayer.py | 33 ++++++++++++++++++++++++++++++++ src/icosagon/convolve.py | 7 +++++++ tests/icosagon/test_convlayer.py | 18 +++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/src/icosagon/convlayer.py b/src/icosagon/convlayer.py index e98b55e..ec0efcf 100644 --- a/src/icosagon/convlayer.py +++ b/src/icosagon/convlayer.py @@ -68,6 +68,33 @@ class DecagonLayer(torch.nn.Module): self.next_layer_repr[fam.node_type_row].append( Convolutions(fam.node_type_column, convolutions)) + # def build_fam_two_node_types_sym(self, fam) -> None: + # convolutions_row = torch.nn.ModuleList() + # convolutions_column = torch.nn.ModuleList() + # + # if self.input_dim[fam.node_type_column] != \ + # self.input_dim[fam.node_type_row]: + # raise ValueError('input_dim for row and column must be equal for a symmetric family') + # + # if self.output_dim[fam.node_type_column] != \ + # self.output_dim[fam.node_type_row]: + # raise ValueError('output_dim for row and column must be equal for a symmetric family') + # + # for r in fam.relation_types: + # assert r.adjacency_matrix is not None and \ + # r.adjacency_matrix_backward is not None + # conv = DropoutGraphConvActivation(self.input_dim[fam.node_type_column], + # self.output_dim[fam.node_type_row], r.adjacency_matrix, + # self.keep_prob, self.rel_activation) + # convolutions_row.append(conv) + # convolutions_column.append(conv.clone(r.adjacency_matrix_backward)) + # + # self.next_layer_repr[fam.node_type_row].append( + # Convolutions(fam.node_type_column, convolutions_row)) + # + # self.next_layer_repr[fam.node_type_column].append( + # Convolutions(fam.node_type_row, convolutions_column)) + def build_fam_two_node_types(self, fam) -> None: convolutions_row = torch.nn.ModuleList() convolutions_column = torch.nn.ModuleList() @@ -91,6 +118,12 @@ class DecagonLayer(torch.nn.Module): self.next_layer_repr[fam.node_type_column].append( Convolutions(fam.node_type_row, convolutions_column)) + # def build_fam_two_node_types(self, fam) -> None: + # if fam.is_symmetric: + # self.build_fam_two_node_types_sym(fam) + # else: + # self.build_fam_two_node_types_asym(fam) + def build_family(self, fam) -> None: if fam.node_type_row == fam.node_type_column: self.build_fam_one_node_type(fam) diff --git a/src/icosagon/convolve.py b/src/icosagon/convolve.py index 364f61a..63cf049 100644 --- a/src/icosagon/convolve.py +++ b/src/icosagon/convolve.py @@ -48,3 +48,10 @@ class DropoutGraphConvActivation(torch.nn.Module): x = self.graph_conv(x) x = self.activation(x) return x + + def clone(self, adjacency_matrix) -> 'DropoutGraphConvActivation': + res = DropoutGraphConvActivation(self.input_dim, + self.output_dim, adjacency_matrix, self.keep_prob, + self.activation) + res.graph_conv.weight = self.graph_conv.weight + return res diff --git a/tests/icosagon/test_convlayer.py b/tests/icosagon/test_convlayer.py index 96fb225..636301f 100644 --- a/tests/icosagon/test_convlayer.py +++ b/tests/icosagon/test_convlayer.py @@ -280,3 +280,21 @@ def test_module_nesting_06(): dummy_7 = Dummy7() dummy_7 = dummy_7.to(device) assert dummy_7.dummy_1[0][0].whatever.device == device + + +def test_parameter_count_01(): + d = Data() + d.add_node_type('Dummy', 100) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, True) + fam.add_relation_type('Dummy Relation 1', + _symmetric_random(100, 100).to_sparse()) + fam.add_relation_type('Dummy Relation 2', + _symmetric_random(100, 100).to_sparse()) + + in_layer = OneHotInputLayer(d) + assert len(list(in_layer.parameters())) == 1 + + d_layer = DecagonLayer(in_layer.output_dim, output_dim=32, data=d, + keep_prob=1., rel_activation=lambda x: x, + layer_activation=lambda x: x) + assert len(list(d_layer.parameters())) == 2 -- 2.26.2 From 094813b29840a27805f036bfef8e1fe69f7bc7e5 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 19 Jun 2020 21:55:10 +0200 Subject: [PATCH 121/227] Change test in test_unigram_03(). --- src/icosagon/convlayer.py | 33 --------------------------------- tests/icosagon/test_sampling.py | 26 +++++++++++--------------- 2 files changed, 11 insertions(+), 48 deletions(-) diff --git a/src/icosagon/convlayer.py b/src/icosagon/convlayer.py index ec0efcf..e98b55e 100644 --- a/src/icosagon/convlayer.py +++ b/src/icosagon/convlayer.py @@ -68,33 +68,6 @@ class DecagonLayer(torch.nn.Module): self.next_layer_repr[fam.node_type_row].append( Convolutions(fam.node_type_column, convolutions)) - # def build_fam_two_node_types_sym(self, fam) -> None: - # convolutions_row = torch.nn.ModuleList() - # convolutions_column = torch.nn.ModuleList() - # - # if self.input_dim[fam.node_type_column] != \ - # self.input_dim[fam.node_type_row]: - # raise ValueError('input_dim for row and column must be equal for a symmetric family') - # - # if self.output_dim[fam.node_type_column] != \ - # self.output_dim[fam.node_type_row]: - # raise ValueError('output_dim for row and column must be equal for a symmetric family') - # - # for r in fam.relation_types: - # assert r.adjacency_matrix is not None and \ - # r.adjacency_matrix_backward is not None - # conv = DropoutGraphConvActivation(self.input_dim[fam.node_type_column], - # self.output_dim[fam.node_type_row], r.adjacency_matrix, - # self.keep_prob, self.rel_activation) - # convolutions_row.append(conv) - # convolutions_column.append(conv.clone(r.adjacency_matrix_backward)) - # - # self.next_layer_repr[fam.node_type_row].append( - # Convolutions(fam.node_type_column, convolutions_row)) - # - # self.next_layer_repr[fam.node_type_column].append( - # Convolutions(fam.node_type_row, convolutions_column)) - def build_fam_two_node_types(self, fam) -> None: convolutions_row = torch.nn.ModuleList() convolutions_column = torch.nn.ModuleList() @@ -118,12 +91,6 @@ class DecagonLayer(torch.nn.Module): self.next_layer_repr[fam.node_type_column].append( Convolutions(fam.node_type_row, convolutions_column)) - # def build_fam_two_node_types(self, fam) -> None: - # if fam.is_symmetric: - # self.build_fam_two_node_types_sym(fam) - # else: - # self.build_fam_two_node_types_asym(fam) - def build_family(self, fam) -> None: if fam.node_type_row == fam.node_type_column: self.build_fam_one_node_type(fam) diff --git a/tests/icosagon/test_sampling.py b/tests/icosagon/test_sampling.py index 8552949..824541b 100644 --- a/tests/icosagon/test_sampling.py +++ b/tests/icosagon/test_sampling.py @@ -118,7 +118,7 @@ def test_unigram_02(): def test_unigram_03(): range_max = 7 distortion = 0.75 - batch_size = 25 + batch_size = 2500 unigrams = [ 1, 3, 2, 1, 2, 1, 3] num_true = 1 @@ -129,8 +129,8 @@ def test_unigram_03(): true_classes_tf = tf.convert_to_tensor(true_classes) true_classes_torch = torch.tensor(true_classes) - counts_tf = defaultdict(list) - counts_torch = defaultdict(list) + counts_tf = torch.zeros(range_max) + counts_torch = torch.zeros(range_max) for i in range(10): neg_samples, _, _ = tf.nn.fixed_unigram_candidate_sampler( @@ -142,29 +142,25 @@ def test_unigram_03(): distortion=distortion, unigrams=unigrams) - counts = defaultdict(int) + counts = torch.zeros(range_max) with tf.Session() as sess: neg_samples = neg_samples.eval() for x in neg_samples: counts[x.item()] += 1 - for k, v in counts.items(): - counts_tf[k].append(v) + counts_tf += counts neg_samples = icosagon.sampling.fixed_unigram_candidate_sampler( true_classes=true_classes, distortion=distortion, unigrams=unigrams) - counts = defaultdict(int) + counts = torch.zeros(range_max) for x in neg_samples: counts[x.item()] += 1 - for k, v in counts.items(): - counts_torch[k].append(v) + counts_torch += counts - for i in range(range_max): - print('counts_tf[%d]:' % i, counts_tf[i]) - print('counts_torch[%d]:' % i, counts_torch[i]) + print('counts_tf:', counts_tf) + print('counts_torch:', counts_torch) - for i in range(range_max): - statistic, pvalue = scipy.stats.ttest_ind(counts_tf[i], counts_torch[i]) - assert pvalue * range_max > .05 + distance = scipy.stats.wasserstein_distance(counts_tf, counts_torch) + assert distance < 2000 -- 2.26.2 From 108140b45e1ad86f9d17ca4ef50e40fa0aaa6165 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 20 Jun 2020 14:53:26 +0200 Subject: [PATCH 122/227] Add test_graph_conv_parameter_count_01(). --- tests/icosagon/test_convolve.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/icosagon/test_convolve.py b/tests/icosagon/test_convolve.py index 5802800..79fb2ac 100644 --- a/tests/icosagon/test_convolve.py +++ b/tests/icosagon/test_convolve.py @@ -188,3 +188,11 @@ def test_dropout_graph_conv_activation_dense_03(): def test_dropout_graph_conv_activation_sparse_03(): _test_dropout_graph_conv_activation_03(True) + + +def test_graph_conv_parameter_count_01(): + adj_mat = torch.rand((10, 20)).round() + + conv = GraphConv(20, 20, adj_mat) + + assert len(list(conv.parameters())) == 1 -- 2.26.2 From b79487e772fbd7dc2f3517518424b7d472339ea5 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 20 Jun 2020 14:54:11 +0200 Subject: [PATCH 123/227] Add test_dropout_graph_conv_activation_parameter_count_01(). --- tests/icosagon/test_convolve.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/icosagon/test_convolve.py b/tests/icosagon/test_convolve.py index 79fb2ac..d4df6ea 100644 --- a/tests/icosagon/test_convolve.py +++ b/tests/icosagon/test_convolve.py @@ -196,3 +196,11 @@ def test_graph_conv_parameter_count_01(): conv = GraphConv(20, 20, adj_mat) assert len(list(conv.parameters())) == 1 + + +def test_dropout_graph_conv_activation_parameter_count_01(): + adj_mat = torch.rand((10, 20)).round() + + conv = DropoutGraphConvActivation(20, 20, adj_mat) + + assert len(list(conv.parameters())) == 1 -- 2.26.2 From 6c8fdb7091f447c53c9c697caba481c6abdc26bf Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 20 Jun 2020 15:00:32 +0200 Subject: [PATCH 124/227] Add test_decode_layer_parameter_count_[01-03](). --- tests/icosagon/test_declayer.py | 53 +++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/icosagon/test_declayer.py b/tests/icosagon/test_declayer.py index 216b1ed..732fd45 100644 --- a/tests/icosagon/test_declayer.py +++ b/tests/icosagon/test_declayer.py @@ -232,3 +232,56 @@ def test_decode_layer_05(): # assert isinstance(rel_pred.edges_neg, TrainValTest) # assert isinstance(rel_pred.edges_back_pos, TrainValTest) # assert isinstance(rel_pred.edges_back_neg, TrainValTest) + + +def test_decode_layer_parameter_count_01(): + d = Data() + d.add_node_type('Dummy', 100) + + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Relation 1', + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) + + dec = DecodeLayer(input_dim=[ 32 ], data=prep_d, keep_prob=1., + activation=lambda x: x) + + assert len(list(dec.parameters())) == 2 + + +def test_decode_layer_parameter_count_02(): + d = Data() + d.add_node_type('Dummy', 100) + + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Relation 1', + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + fam.add_relation_type('Dummy Relation 2', + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) + + dec = DecodeLayer(input_dim=[ 32 ], data=prep_d, keep_prob=1., + activation=lambda x: x) + + assert len(list(dec.parameters())) == 3 + + +def test_decode_layer_parameter_count_03(): + d = Data() + d.add_node_type('Dummy', 100) + + for _ in range(2): + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Relation 1', + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + fam.add_relation_type('Dummy Relation 2', + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) + + dec = DecodeLayer(input_dim=[ 32 ], data=prep_d, keep_prob=1., + activation=lambda x: x) + + assert len(list(dec.parameters())) == 6 -- 2.26.2 From 77c52d8543ed71597ca9f034a0e5399bf78f120f Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 20 Jun 2020 15:06:08 +0200 Subject: [PATCH 125/227] Add test_[dedicom,dist_mult,bilinear,inner_product]_decoder_parameter_count_01(). --- tests/icosagon/test_decode.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/icosagon/test_decode.py b/tests/icosagon/test_decode.py index 96ca3d7..b8c9cea 100644 --- a/tests/icosagon/test_decode.py +++ b/tests/icosagon/test_decode.py @@ -210,3 +210,31 @@ def test_empty_inner_product_decoder_01(): for i in range(len(res)): assert res[i].shape == (0,) + + +def test_dedicom_decoder_parameter_count_01(): + dec = DEDICOMDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + + assert len(list(dec.parameters())) == 8 + + +def test_dist_mult_decoder_parameter_count_01(): + dec = DistMultDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + + assert len(list(dec.parameters())) == 7 + + +def test_bilinear_decoder_parameter_count_01(): + dec = BilinearDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + + assert len(list(dec.parameters())) == 7 + + +def test_inner_product_decoder_parameter_count_01(): + dec = InnerProductDecoder(32, 7, keep_prob=1., + activation=torch.sigmoid) + + assert len(list(dec.parameters())) == 0 -- 2.26.2 From 1b72e41ea9a1822123cf718c0f410ae86a62fec4 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 20 Jun 2020 15:18:48 +0200 Subject: [PATCH 126/227] Add test_[one_hot_]input_layer_parameter_count_01(). --- tests/icosagon/test_input.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/icosagon/test_input.py b/tests/icosagon/test_input.py index 8bb5dc0..1ea676a 100644 --- a/tests/icosagon/test_input.py +++ b/tests/icosagon/test_input.py @@ -107,3 +107,16 @@ def test_one_hot_input_layer_04(): layer = OneHotInputLayer(d) s = repr(layer) assert s.startswith('Icosagon one-hot input layer') + + +def test_one_hot_input_layer_parameter_count_01(): + d = _some_data() + layer = OneHotInputLayer(d) + assert len(list(layer.parameters())) == 2 + + +def test_input_layer_parameter_count_01(): + d = _some_data() + for output_dim in [32, 64, 128]: + layer = InputLayer(d, output_dim) + assert len(list(layer.parameters())) == 2 -- 2.26.2 From a6de8c784604aebf299c478135c0a25a4c3cadfd Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 20 Jun 2020 17:13:31 +0200 Subject: [PATCH 127/227] Add test_flatten_predictions_05(). --- tests/icosagon/test_batch.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/icosagon/test_batch.py b/tests/icosagon/test_batch.py index b6cd6d6..b5882db 100644 --- a/tests/icosagon/test_batch.py +++ b/tests/icosagon/test_batch.py @@ -92,6 +92,27 @@ def test_flatten_predictions_04(): pred_flat = flatten_predictions(pred, part_type='x') +def test_flatten_predictions_05(): + x = torch.rand(5000) + y = torch.cat([ x, x ]) + z = torch.cat([ torch.ones(5000), torch.zeros(5000) ]) + + rel_pred = RelationPredictions( + TrainValTest(x, torch.zeros(0), torch.zeros(0)), + TrainValTest(x, torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)), + TrainValTest(torch.zeros(0), torch.zeros(0), torch.zeros(0)) + ) + fam_pred = RelationFamilyPredictions([ rel_pred ]) + pred = Predictions([ fam_pred ]) + + for _ in range(10): + pred_flat = flatten_predictions(pred, part_type='train') + assert torch.all(pred_flat.predictions == y) + assert torch.all(pred_flat.truth == z) + assert pred_flat.part_type == 'train' + + def test_batch_indices_01(): indices = BatchIndices(torch.tensor([0, 1, 2, 3, 4]), 'train') assert torch.all(indices.indices == torch.tensor([0, 1, 2, 3, 4])) -- 2.26.2 From 2a2aecb3674ad9db0a1662dcf6beed3fb7e806c6 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 20 Jun 2020 17:50:13 +0200 Subject: [PATCH 128/227] Add test_prep_rel_one_node_type_01(). --- tests/icosagon/test_trainprep.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/icosagon/test_trainprep.py b/tests/icosagon/test_trainprep.py index b3a2d1f..2efcf45 100644 --- a/tests/icosagon/test_trainprep.py +++ b/tests/icosagon/test_trainprep.py @@ -8,7 +8,8 @@ from icosagon.trainprep import TrainValTest, \ train_val_test_split_edges, \ get_edges_and_degrees, \ prepare_adj_mat, \ - prepare_relation_type + prepare_relation_type, \ + prep_rel_one_node_type import torch import pytest import numpy as np @@ -110,6 +111,34 @@ def test_prepare_relation_type_01(): _ = prepare_relation_type(r, ratios, False) +def test_prep_rel_one_node_type_01(): + adj_mat = torch.zeros(100) + perm = torch.randperm(100) + adj_mat[perm[:10]] = 1 + adj_mat = adj_mat.view(10, 10) + rel = RelationType('Dummy Relation', 0, 0, adj_mat, None) + ratios = TrainValTest(.8, .1, .1) + prep_rel = prep_rel_one_node_type(rel, ratios) + assert prep_rel.name == rel.name + assert prep_rel.node_type_row == rel.node_type_row + assert prep_rel.node_type_column == rel.node_type_column + assert prep_rel.adjacency_matrix.shape == rel.adjacency_matrix.shape + assert prep_rel.adjacency_matrix_backward is None + assert len(prep_rel.edges_pos.train) == 8 + assert len(prep_rel.edges_pos.val) == 1 + assert len(prep_rel.edges_pos.test) == 1 + assert len(prep_rel.edges_neg.train) == 8 + assert len(prep_rel.edges_neg.val) == 1 + assert len(prep_rel.edges_neg.test) == 1 + + assert len(prep_rel.edges_back_pos.train) == 0 + assert len(prep_rel.edges_back_pos.val) == 0 + assert len(prep_rel.edges_back_pos.test) == 0 + assert len(prep_rel.edges_back_neg.train) == 0 + assert len(prep_rel.edges_back_neg.val) == 0 + assert len(prep_rel.edges_back_neg.test) == 0 + + # def prepare_relation(r, ratios): # adj_mat = r.adjacency_matrix -- 2.26.2 From 5c24883ba4eaf4df27bd03ea72e547d7b1ea6c76 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 20 Jun 2020 17:56:20 +0200 Subject: [PATCH 129/227] Add test_prep_rel_two_node_types_sym_01(). --- tests/icosagon/test_trainprep.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/icosagon/test_trainprep.py b/tests/icosagon/test_trainprep.py index 2efcf45..79a8aee 100644 --- a/tests/icosagon/test_trainprep.py +++ b/tests/icosagon/test_trainprep.py @@ -9,7 +9,8 @@ from icosagon.trainprep import TrainValTest, \ get_edges_and_degrees, \ prepare_adj_mat, \ prepare_relation_type, \ - prep_rel_one_node_type + prep_rel_one_node_type, \ + prep_rel_two_node_types_sym import torch import pytest import numpy as np @@ -139,6 +140,33 @@ def test_prep_rel_one_node_type_01(): assert len(prep_rel.edges_back_neg.test) == 0 +def test_prep_rel_two_node_types_sym_01(): + adj_mat = torch.zeros(200) + perm = torch.randperm(100) + adj_mat[perm[:10]] = 1 + adj_mat = adj_mat.view(10, 20) + rel = RelationType('Dummy Relation', 0, 1, adj_mat, None) + ratios = TrainValTest(.8, .1, .1) + prep_rel = prep_rel_two_node_types_sym(rel, ratios) + assert prep_rel.name == rel.name + assert prep_rel.node_type_row == rel.node_type_row + assert prep_rel.node_type_column == rel.node_type_column + assert prep_rel.adjacency_matrix.shape == rel.adjacency_matrix.shape + assert prep_rel.adjacency_matrix_backward.shape == (20, 10) + assert len(prep_rel.edges_pos.train) == 8 + assert len(prep_rel.edges_pos.val) == 1 + assert len(prep_rel.edges_pos.test) == 1 + assert len(prep_rel.edges_neg.train) == 8 + assert len(prep_rel.edges_neg.val) == 1 + assert len(prep_rel.edges_neg.test) == 1 + + assert len(prep_rel.edges_back_pos.train) == 0 + assert len(prep_rel.edges_back_pos.val) == 0 + assert len(prep_rel.edges_back_pos.test) == 0 + assert len(prep_rel.edges_back_neg.train) == 0 + assert len(prep_rel.edges_back_neg.val) == 0 + assert len(prep_rel.edges_back_neg.test) == 0 + # def prepare_relation(r, ratios): # adj_mat = r.adjacency_matrix -- 2.26.2 From e2a57e840c880ca0615ff1054a13ab134ea88c5b Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 20 Jun 2020 18:00:15 +0200 Subject: [PATCH 130/227] Add test_prep_rel_two_node_types_asym_01(). --- tests/icosagon/test_trainprep.py | 37 +++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/tests/icosagon/test_trainprep.py b/tests/icosagon/test_trainprep.py index 79a8aee..eb78859 100644 --- a/tests/icosagon/test_trainprep.py +++ b/tests/icosagon/test_trainprep.py @@ -10,7 +10,8 @@ from icosagon.trainprep import TrainValTest, \ prepare_adj_mat, \ prepare_relation_type, \ prep_rel_one_node_type, \ - prep_rel_two_node_types_sym + prep_rel_two_node_types_sym, \ + prep_rel_two_node_types_asym import torch import pytest import numpy as np @@ -168,6 +169,40 @@ def test_prep_rel_two_node_types_sym_01(): assert len(prep_rel.edges_back_neg.test) == 0 +def test_prep_rel_two_node_types_asym_01(): + adj_mat = torch.zeros(200) + perm = torch.randperm(100) + adj_mat[perm[:10]] = 1 + adj_mat = adj_mat.view(10, 20) + + adj_mat_back = torch.zeros(200) + perm = torch.randperm(100) + adj_mat_back[perm[:10]] = 1 + adj_mat_back = adj_mat_back.view(20, 10) + + rel = RelationType('Dummy Relation', 0, 1, adj_mat, adj_mat_back) + ratios = TrainValTest(.8, .1, .1) + prep_rel = prep_rel_two_node_types_asym(rel, ratios) + assert prep_rel.name == rel.name + assert prep_rel.node_type_row == rel.node_type_row + assert prep_rel.node_type_column == rel.node_type_column + assert prep_rel.adjacency_matrix.shape == rel.adjacency_matrix.shape + assert prep_rel.adjacency_matrix_backward.shape == rel.adjacency_matrix_backward.shape + assert len(prep_rel.edges_pos.train) == 8 + assert len(prep_rel.edges_pos.val) == 1 + assert len(prep_rel.edges_pos.test) == 1 + assert len(prep_rel.edges_neg.train) == 8 + assert len(prep_rel.edges_neg.val) == 1 + assert len(prep_rel.edges_neg.test) == 1 + + assert len(prep_rel.edges_back_pos.train) == 8 + assert len(prep_rel.edges_back_pos.val) == 1 + assert len(prep_rel.edges_back_pos.test) == 1 + assert len(prep_rel.edges_back_neg.train) == 8 + assert len(prep_rel.edges_back_neg.val) == 1 + assert len(prep_rel.edges_back_neg.test) == 1 + + # def prepare_relation(r, ratios): # adj_mat = r.adjacency_matrix # adj_mat_train, edges_pos, edges_neg = prepare_adj_mat(adj_mat) -- 2.26.2 From 2cb561b1d44810d1e861777c398acd4359416e43 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 20 Jun 2020 18:16:49 +0200 Subject: [PATCH 131/227] Add test_prepare_relation_type_[01,02](). --- tests/icosagon/test_trainprep.py | 52 ++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/icosagon/test_trainprep.py b/tests/icosagon/test_trainprep.py index eb78859..73ec624 100644 --- a/tests/icosagon/test_trainprep.py +++ b/tests/icosagon/test_trainprep.py @@ -17,6 +17,7 @@ import pytest import numpy as np from itertools import chain from icosagon.data import RelationType +import icosagon.trainprep def test_train_val_test_split_edges_01(): @@ -203,6 +204,57 @@ def test_prep_rel_two_node_types_asym_01(): assert len(prep_rel.edges_back_neg.test) == 1 +def test_prepare_relation_type_01(): + with pytest.raises(ValueError): + prepare_relation_type(None, None, True) + + adj_mat = torch.rand(10, 10).round() + rel = RelationType('Dummy Relation', 0, 0, adj_mat, None) + with pytest.raises(ValueError): + prepare_relation_type(rel, None, True) + + ratios = TrainValTest(.8, .1, .1) + with pytest.raises(ValueError): + prepare_relation_type(None, ratios, True) + + _ = prepare_relation_type(rel, ratios, True) + + +def test_prepare_relation_type_02(monkeypatch): + a = 0 + b = 0 + c = 0 + def fake_prep_rel_one_node_type(*args, **kwargs): + nonlocal a + a += 1 + def fake_prep_rel_two_node_types_sym(*args, **kwargs): + nonlocal b + b += 1 + def fake_prep_rel_two_node_types_asym(*args, **kwargs): + nonlocal c + c += 1 + monkeypatch.setattr(icosagon.trainprep, 'prep_rel_one_node_type', + fake_prep_rel_one_node_type) + monkeypatch.setattr(icosagon.trainprep, 'prep_rel_two_node_types_sym', + fake_prep_rel_two_node_types_sym) + monkeypatch.setattr(icosagon.trainprep, 'prep_rel_two_node_types_asym', + fake_prep_rel_two_node_types_asym) + ratios = TrainValTest(.8, .1, .1) + rel = RelationType('Dummy Relation', 0, 0, None, None) + prepare_relation_type(rel, ratios, False) + assert a == 1 + rel = RelationType('Dummy Relation', 0, 0, None, None) + prepare_relation_type(rel, ratios, True) + assert a == 2 + rel = RelationType('Dummy Relation', 0, 1, None, None) + prepare_relation_type(rel, ratios, True) + assert b == 1 + rel = RelationType('Dummy Relation', 0, 1, None, None) + prepare_relation_type(rel, ratios, False) + assert c == 1 + assert a == 2 and b == 1 and c == 1 + + # def prepare_relation(r, ratios): # adj_mat = r.adjacency_matrix # adj_mat_train, edges_pos, edges_neg = prepare_adj_mat(adj_mat) -- 2.26.2 From 1f977eee4c7b5e65950aecdef1685e4669417228 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Sat, 20 Jun 2020 18:18:38 +0200 Subject: [PATCH 132/227] Fix test names in test_trainprep. --- tests/icosagon/test_trainprep.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/icosagon/test_trainprep.py b/tests/icosagon/test_trainprep.py index 73ec624..b49646b 100644 --- a/tests/icosagon/test_trainprep.py +++ b/tests/icosagon/test_trainprep.py @@ -204,7 +204,7 @@ def test_prep_rel_two_node_types_asym_01(): assert len(prep_rel.edges_back_neg.test) == 1 -def test_prepare_relation_type_01(): +def test_prepare_relation_type_02(): with pytest.raises(ValueError): prepare_relation_type(None, None, True) @@ -220,7 +220,7 @@ def test_prepare_relation_type_01(): _ = prepare_relation_type(rel, ratios, True) -def test_prepare_relation_type_02(monkeypatch): +def test_prepare_relation_type_03(monkeypatch): a = 0 b = 0 c = 0 -- 2.26.2 From 59edeaa915927bd2b5bdb60a4a261abb954bb073 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 15 Jul 2020 16:58:03 +0200 Subject: [PATCH 133/227] Add docker/. --- docker/mlflow-tracking-server/Dockerfile | 33 ++++++++++++++++++++++++ docker/postgresql/Dockerfile | 3 +++ 2 files changed, 36 insertions(+) create mode 100644 docker/mlflow-tracking-server/Dockerfile create mode 100644 docker/postgresql/Dockerfile diff --git a/docker/mlflow-tracking-server/Dockerfile b/docker/mlflow-tracking-server/Dockerfile new file mode 100644 index 0000000..735a0d0 --- /dev/null +++ b/docker/mlflow-tracking-server/Dockerfile @@ -0,0 +1,33 @@ +FROM debian:latest + +RUN apt-get update && \ + apt-get install -y python3-pip \ + python3-pandas \ + python3-alembic \ + python3-sqlalchemy \ + python3-yaml \ + python3-flask \ + python3-gunicorn \ + python3-protobuf \ + python3-urllib3 \ + python3-certifi \ + python3-idna \ + python3-requests \ +# python3-docker \ + python3-smmap \ + python3-gitdb \ + python3-git \ + python3-sqlparse \ + python3-oauthlib \ + python3-requests-oauthlib \ + python3-isodate \ +# python3-msrest \ + python3-prometheus-client \ + python3-cloudpickle \ + python3-tabulate && \ + pip3 install mlflow + +# RUN apk add --no-cache gcc gfortran libgfortran musl-dev python3 py3 +#pip python3-dev py3-numpy libffi libffi-dev && \ +# pip3 install mlflow && \ +# apk del musl-dev python3-dev gcc gfortran py3-pip libffi-dev diff --git a/docker/postgresql/Dockerfile b/docker/postgresql/Dockerfile new file mode 100644 index 0000000..dcfa052 --- /dev/null +++ b/docker/postgresql/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:latest + +RUN apk add postgresql -- 2.26.2 From 373a12f51a0f18affd2c4e20bebcbf5af69a65a8 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 16 Jul 2020 12:44:48 +0200 Subject: [PATCH 134/227] Start working on experiments/decagon_run. --- docker/postgresql/Dockerfile | 3 ++ docker/postgresql/docker-init.sh | 5 +++ experiments/decagon_run/decagon_run.py | 62 ++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 docker/postgresql/docker-init.sh create mode 100644 experiments/decagon_run/decagon_run.py diff --git a/docker/postgresql/Dockerfile b/docker/postgresql/Dockerfile index dcfa052..137526d 100644 --- a/docker/postgresql/Dockerfile +++ b/docker/postgresql/Dockerfile @@ -1,3 +1,6 @@ FROM alpine:latest RUN apk add postgresql + +RUN mkdir /data && \ + chown postgres:postgres /data diff --git a/docker/postgresql/docker-init.sh b/docker/postgresql/docker-init.sh new file mode 100644 index 0000000..a69650a --- /dev/null +++ b/docker/postgresql/docker-init.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ ! -f /data/PG_VERSION ]; then + initdb -D /data --pwfile=/superuser_password +fi diff --git a/experiments/decagon_run/decagon_run.py b/experiments/decagon_run/decagon_run.py new file mode 100644 index 0000000..239250e --- /dev/null +++ b/experiments/decagon_run/decagon_run.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +from icosagon.data import Data +import os +import pandas as pd +from bisect import bisect_left +import torch + + +def index(a, x): + i = bisect_left(a, x) + if i != len(a) and a[i] == x: + return i + raise ValueError + + +def main(): + path = '/pstore/data/data_science/ref/decagon' + df_combo = pd.read_csv(os.path.join(path, 'bio-decagon-combo.csv')) + df_effcat = pd.read_csv(os.path.join(path, 'bio-decagon-effectcategories.csv')) + df_mono = pd.read_csv(os.path.join(path, 'bio-decagon-mono.csv')) + df_ppi = pd.read_csv(os.path.join(path, 'bio-decagon-ppi.csv')) + df_tgtall = pd.read_csv(os.path.join(path, 'bio-decagon-targets-all.csv')) + df_tgt = pd.read_csv(os.path.join(path, 'bio-decagon-targets.csv')) + lst = [ 'df_combo', 'df_effcat', 'df_mono', 'df_ppi', 'df_tgtall', 'df_tgt' ] + for nam in lst: + print(f'len({nam}): {len(locals()[nam])}') + print(f'{nam}.columns: {locals()[nam].columns}') + + genes = set() + genes = genes.union(df_ppi['Gene 1']).union(df_ppi['Gene 2']) \ + .union(df_tgtall['Gene']).union(df_tgt['Gene']) + genes = sorted(genes) + print('len(genes):', len(genes)) + + drugs = set() + drugs = drugs.union(df_combo['STITCH 1']).union(df_combo['STITCH 2']) \ + .union(df_mono['STITCH']).union(df_tgtall['STITCH']).union(df_tgt['STITCH']) + drugs = sorted(drugs) + print('len(drugs):', len(drugs)) + + data = Data() + data.add_node_type('Gene', len(genes)) + data.add_node_type('Drug', len(drugs)) + + print('Indexing rows...') + rows = [index(genes, g) for g in df_ppi['Gene 1']] + print('Indexing cols...') + cols = [index(genes, g) for g in df_ppi['Gene 2']] + indices = list(zip(rows, cols)) + indices = torch.tensor(indices).transpose(0, 1) + values = torch.ones(len(rows)) + print('indices.shape:', indices.shape, 'values.shape:', values.shape) + adj_mat = torch.sparse_coo_tensor(indices, values, size=(len(genes),) * 2) + adj_mat = (adj_mat + adj_mat.transpose(0, 1)) / 2 + print('adj_mat created') + fam = data.add_relation_family('PPI', 0, 0, True) + rel = fam.add_relation_type('PPI', adj_mat) + + +if __name__ == '__main__': + main() -- 2.26.2 From d4dab52e82743fc9d5bd9d6268e818d696f2b693 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 16 Jul 2020 14:08:37 +0200 Subject: [PATCH 135/227] Make _equal() much faster. --- src/icosagon/data.py | 25 ++++++++++++++++++++----- tests/icosagon/test_data.py | 6 +++++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/icosagon/data.py b/src/icosagon/data.py index 7e39db7..70963ec 100644 --- a/src/icosagon/data.py +++ b/src/icosagon/data.py @@ -14,6 +14,7 @@ from typing import List, \ Type from .decode import DEDICOMDecoder, \ BilinearDecoder +import numpy as np def _equal(x: torch.Tensor, y: torch.Tensor): @@ -23,15 +24,29 @@ def _equal(x: torch.Tensor, y: torch.Tensor): if not x.is_sparse: return (x == y) + # if x.shape != y.shape: + # return torch.tensor(0, dtype=torch.uint8) + + return ((x - y).coalesce().values() == 0) + x = x.coalesce() - indices_x = list(map(tuple, x.indices().transpose(0, 1))) - order_x = sorted(range(len(indices_x)), key=lambda idx: indices_x[idx]) + indices_x = np.empty(x.indices().shape[1], dtype=np.object) + indices_x[:] = list(map(tuple, x.indices().transpose(0, 1))) + order_x = np.argsort(indices_x) + #order_x = sorted(range(len(indices_x)), key=lambda idx: indices_x[idx]) y = y.coalesce() - indices_y = list(map(tuple, y.indices().transpose(0, 1))) - order_y = sorted(range(len(indices_y)), key=lambda idx: indices_y[idx]) + indices_y = np.empty(y.indices().shape[1], dtype=np.object) + indices_y[:] = list(map(tuple, y.indices().transpose(0, 1))) + order_y = np.argsort(indices_y) + # order_y = sorted(range(len(indices_y)), key=lambda idx: indices_y[idx]) + + # print(indices_x.shape, indices_y.shape) + + if not len(indices_x) == len(indices_y): + return torch.tensor(0, dtype=torch.uint8) - if not indices_x == indices_y: + if not np.all(indices_x[order_x] == indices_y[order_y]): return torch.tensor(0, dtype=torch.uint8) return (x.values()[order_x] == y.values()[order_y]) diff --git a/tests/icosagon/test_data.py b/tests/icosagon/test_data.py index 08ded89..57060e9 100644 --- a/tests/icosagon/test_data.py +++ b/tests/icosagon/test_data.py @@ -16,11 +16,15 @@ def test_equal_01(): x = torch.rand((10, 10)) y = torch.rand((10, 10)).round().to_sparse() + print('x == x ?') assert torch.all(_equal(x, x)) + print('y == y ?') assert torch.all(_equal(y, y)) + print('x == y ?') with pytest.raises(ValueError): _equal(x, y) + print('y == z ?') z = torch.rand((10, 10)).round().to_sparse() assert not torch.all(_equal(y, z)) @@ -71,7 +75,7 @@ def test_relation_family_03(): d.add_node_type('B', 5) fam = RelationFamily(d, 'A-B', 0, 1, True, DEDICOMDecoder) - + fam.add_relation_type('A-B', torch.rand((10, 5)).round()) assert torch.all(fam.relation_types[0].adjacency_matrix.transpose(0, 1) == \ -- 2.26.2 From 8aafb6fa018f3d1d5b18120ccb03bc75d574dd32 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 16 Jul 2020 14:09:20 +0200 Subject: [PATCH 136/227] Clean up _equal(). --- src/icosagon/data.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/src/icosagon/data.py b/src/icosagon/data.py index 70963ec..4505adf 100644 --- a/src/icosagon/data.py +++ b/src/icosagon/data.py @@ -24,33 +24,8 @@ def _equal(x: torch.Tensor, y: torch.Tensor): if not x.is_sparse: return (x == y) - # if x.shape != y.shape: - # return torch.tensor(0, dtype=torch.uint8) - return ((x - y).coalesce().values() == 0) - x = x.coalesce() - indices_x = np.empty(x.indices().shape[1], dtype=np.object) - indices_x[:] = list(map(tuple, x.indices().transpose(0, 1))) - order_x = np.argsort(indices_x) - #order_x = sorted(range(len(indices_x)), key=lambda idx: indices_x[idx]) - - y = y.coalesce() - indices_y = np.empty(y.indices().shape[1], dtype=np.object) - indices_y[:] = list(map(tuple, y.indices().transpose(0, 1))) - order_y = np.argsort(indices_y) - # order_y = sorted(range(len(indices_y)), key=lambda idx: indices_y[idx]) - - # print(indices_x.shape, indices_y.shape) - - if not len(indices_x) == len(indices_y): - return torch.tensor(0, dtype=torch.uint8) - - if not np.all(indices_x[order_x] == indices_y[order_y]): - return torch.tensor(0, dtype=torch.uint8) - - return (x.values()[order_x] == y.values()[order_y]) - @dataclass class NodeType(object): -- 2.26.2 From a34a0f5f2ad1534d85c4086e7db0f8be4cea6e65 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 16 Jul 2020 15:15:40 +0200 Subject: [PATCH 137/227] Done data loading in decagon_run. --- experiments/decagon_run/decagon_run.py | 37 +++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/experiments/decagon_run/decagon_run.py b/experiments/decagon_run/decagon_run.py index 239250e..d21cce0 100644 --- a/experiments/decagon_run/decagon_run.py +++ b/experiments/decagon_run/decagon_run.py @@ -5,6 +5,7 @@ import os import pandas as pd from bisect import bisect_left import torch +import sys def index(a, x): @@ -14,7 +15,7 @@ def index(a, x): raise ValueError -def main(): +def load_data(): path = '/pstore/data/data_science/ref/decagon' df_combo = pd.read_csv(os.path.join(path, 'bio-decagon-combo.csv')) df_effcat = pd.read_csv(os.path.join(path, 'bio-decagon-effectcategories.csv')) @@ -43,6 +44,7 @@ def main(): data.add_node_type('Gene', len(genes)) data.add_node_type('Drug', len(drugs)) + print('Preparing PPI...') print('Indexing rows...') rows = [index(genes, g) for g in df_ppi['Gene 1']] print('Indexing cols...') @@ -56,6 +58,39 @@ def main(): print('adj_mat created') fam = data.add_relation_family('PPI', 0, 0, True) rel = fam.add_relation_type('PPI', adj_mat) + print('OK') + + print('Preparing Drug-Gene (Target) edges...') + rows = [index(drugs, d) for d in df_tgtall['STITCH']] + cols = [index(genes, g) for g in df_tgtall['Gene']] + indices = list(zip(rows, cols)) + indices = torch.tensor(indices).transpose(0, 1) + values = torch.ones(len(rows)) + adj_mat = torch.sparse_coo_tensor(indices, values, size=(len(drugs), len(genes))) + fam = data.add_relation_family('Drug-Gene (Target)', 1, 0, True) + rel = fam.add_relation_type('Drug-Gene (Target)', adj_mat) + print('OK') + + print('Preparing Drug-Drug (Side Effect) edges...') + fam = data.add_relation_family('Drug-Drug (Side Effect)', 1, 1, True) + print('# of side effects:', len(df_combo), 'unique:', len(df_combo['Polypharmacy Side Effect'].unique())) + for eff, df in df_combo.groupby('Polypharmacy Side Effect'): + sys.stdout.write('.') # print(eff, '...') + sys.stdout.flush() + rows = [index(drugs, d) for d in df['STITCH 1']] + cols = [index(drugs, d) for d in df['STITCH 2']] + indices = list(zip(rows, cols)) + indices = torch.tensor(indices).transpose(0, 1) + values = torch.ones(len(rows)) + adj_mat = torch.sparse_coo_tensor(indices, values, size=(len(drugs), len(drugs))) + adj_mat = (adj_mat + adj_mat.transpose(0, 1)) / 2 + rel = fam.add_relation_type(df['Polypharmacy Side Effect'], adj_mat) + print() + print('OK') + + +def main(): + data = load_data() if __name__ == '__main__': -- 2.26.2 From 055b4b369d0f85bbb92dfdeda6ea7c81e2690e7d Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 16 Jul 2020 16:49:19 +0200 Subject: [PATCH 138/227] run_epoch() progresses at a crawl. --- experiments/decagon_run/decagon_run.py | 23 +++++++++++++++++++++++ src/icosagon/trainloop.py | 3 ++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/experiments/decagon_run/decagon_run.py b/experiments/decagon_run/decagon_run.py index d21cce0..a490628 100644 --- a/experiments/decagon_run/decagon_run.py +++ b/experiments/decagon_run/decagon_run.py @@ -1,6 +1,10 @@ #!/usr/bin/env python3 from icosagon.data import Data +from icosagon.trainprep import TrainValTest, \ + prepare_training +from icosagon.model import Model +from icosagon.trainloop import TrainLoop import os import pandas as pd from bisect import bisect_left @@ -88,9 +92,28 @@ def load_data(): print() print('OK') + return data + + +def _wrap(obj, method_name): + orig_fn = getattr(obj, method_name) + def fn(*args, **kwargs): + print(f'{method_name}() :: ENTER') + res = orig_fn(*args, **kwargs) + print(f'{method_name}() :: EXIT') + return res + setattr(obj, method_name, fn) + def main(): data = load_data() + prep_d = prepare_training(data, TrainValTest(.8, .1, .1)) + _wrap(Model, 'build') + model = Model(prep_d) + _wrap(TrainLoop, 'build') + _wrap(TrainLoop, 'run_epoch') + loop = TrainLoop(model, batch_size=1000000) + loop.run_epoch() if __name__ == '__main__': diff --git a/src/icosagon/trainloop.py b/src/icosagon/trainloop.py index 5492878..3392a40 100644 --- a/src/icosagon/trainloop.py +++ b/src/icosagon/trainloop.py @@ -46,7 +46,8 @@ class TrainLoop(object): # pred = self.model(None) # n = len(list(iter(batch))) loss_sum = 0 - for indices in batch: + for i, indices in enumerate(batch): + print('%.2f%% (%d/%d)' % (i * batch.batch_size * 100 / batch.total_edge_count, i * batch.batch_size, batch.total_edge_count)) self.opt.zero_grad() pred = self.model(None) pred = flatten_predictions(pred) -- 2.26.2 From 0a0a524dc81e46d9dcea7acface03b0d285a2536 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 16 Jul 2020 18:41:40 +0200 Subject: [PATCH 139/227] Fixes for the GPU. --- experiments/decagon_run/decagon_run.py | 16 ++++++++----- src/icosagon/convolve.py | 2 ++ src/icosagon/normalize.py | 13 ++++++----- src/icosagon/sampling.py | 5 +++++ src/icosagon/trainprep.py | 11 +++++---- tests/icosagon/test_trainloop.py | 31 ++++++++++++++++++++++++++ 6 files changed, 64 insertions(+), 14 deletions(-) diff --git a/experiments/decagon_run/decagon_run.py b/experiments/decagon_run/decagon_run.py index a490628..911caaa 100644 --- a/experiments/decagon_run/decagon_run.py +++ b/experiments/decagon_run/decagon_run.py @@ -19,7 +19,7 @@ def index(a, x): raise ValueError -def load_data(): +def load_data(dev): path = '/pstore/data/data_science/ref/decagon' df_combo = pd.read_csv(os.path.join(path, 'bio-decagon-combo.csv')) df_effcat = pd.read_csv(os.path.join(path, 'bio-decagon-effectcategories.csv')) @@ -57,7 +57,8 @@ def load_data(): indices = torch.tensor(indices).transpose(0, 1) values = torch.ones(len(rows)) print('indices.shape:', indices.shape, 'values.shape:', values.shape) - adj_mat = torch.sparse_coo_tensor(indices, values, size=(len(genes),) * 2) + adj_mat = torch.sparse_coo_tensor(indices, values, size=(len(genes),) * 2, + device=dev) adj_mat = (adj_mat + adj_mat.transpose(0, 1)) / 2 print('adj_mat created') fam = data.add_relation_family('PPI', 0, 0, True) @@ -70,7 +71,8 @@ def load_data(): indices = list(zip(rows, cols)) indices = torch.tensor(indices).transpose(0, 1) values = torch.ones(len(rows)) - adj_mat = torch.sparse_coo_tensor(indices, values, size=(len(drugs), len(genes))) + adj_mat = torch.sparse_coo_tensor(indices, values, size=(len(drugs), len(genes)), + device=dev) fam = data.add_relation_family('Drug-Gene (Target)', 1, 0, True) rel = fam.add_relation_type('Drug-Gene (Target)', adj_mat) print('OK') @@ -86,7 +88,8 @@ def load_data(): indices = list(zip(rows, cols)) indices = torch.tensor(indices).transpose(0, 1) values = torch.ones(len(rows)) - adj_mat = torch.sparse_coo_tensor(indices, values, size=(len(drugs), len(drugs))) + adj_mat = torch.sparse_coo_tensor(indices, values, size=(len(drugs), len(drugs)), + device=dev) adj_mat = (adj_mat + adj_mat.transpose(0, 1)) / 2 rel = fam.add_relation_type(df['Polypharmacy Side Effect'], adj_mat) print() @@ -106,10 +109,13 @@ def _wrap(obj, method_name): def main(): - data = load_data() + dev = torch.device('cuda:0') + data = load_data(dev) prep_d = prepare_training(data, TrainValTest(.8, .1, .1)) _wrap(Model, 'build') model = Model(prep_d) + model = model.to(dev) + # model = torch.nn.DataParallel(model, ['cuda:0', 'cuda:1']) _wrap(TrainLoop, 'build') _wrap(TrainLoop, 'run_epoch') loop = TrainLoop(model, batch_size=1000000) diff --git a/src/icosagon/convolve.py b/src/icosagon/convolve.py index 63cf049..09bd32a 100644 --- a/src/icosagon/convolve.py +++ b/src/icosagon/convolve.py @@ -8,6 +8,7 @@ import torch from .dropout import dropout from .weights import init_glorot from typing import List, Callable +import pdb class GraphConv(torch.nn.Module): @@ -44,6 +45,7 @@ class DropoutGraphConvActivation(torch.nn.Module): self.graph_conv = GraphConv(input_dim, output_dim, adjacency_matrix) def forward(self, x: torch.Tensor) -> torch.Tensor: + # pdb.set_trace() x = dropout(x, self.keep_prob) x = self.graph_conv(x) x = self.activation(x) diff --git a/src/icosagon/normalize.py b/src/icosagon/normalize.py index ae63c51..b41dfb1 100644 --- a/src/icosagon/normalize.py +++ b/src/icosagon/normalize.py @@ -44,9 +44,11 @@ def add_eye_sparse(adj_mat: torch.Tensor) -> torch.Tensor: indices = adj_mat.indices() values = adj_mat.values() - eye_indices = torch.arange(adj_mat.shape[0], dtype=indices.dtype).view(1, -1) + eye_indices = torch.arange(adj_mat.shape[0], dtype=indices.dtype, + device=adj_mat.device).view(1, -1) eye_indices = torch.cat((eye_indices, eye_indices), 0) - eye_values = torch.ones(adj_mat.shape[0], dtype=values.dtype) + eye_values = torch.ones(adj_mat.shape[0], dtype=values.dtype, + device=adj_mat.device) indices = torch.cat((indices, eye_indices), 1) values = torch.cat((values, eye_values), 0) @@ -72,7 +74,8 @@ def norm_adj_mat_one_node_type_dense(adj_mat: torch.Tensor) -> torch.Tensor: _check_dense(adj_mat) _check_square(adj_mat) - adj_mat = adj_mat + torch.eye(adj_mat.shape[0], dtype=adj_mat.dtype) + adj_mat = adj_mat + torch.eye(adj_mat.shape[0], dtype=adj_mat.dtype, + device=adj_mat.device) adj_mat = norm_adj_mat_two_node_types_dense(adj_mat) return adj_mat @@ -96,9 +99,9 @@ def norm_adj_mat_two_node_types_sparse(adj_mat: torch.Tensor) -> torch.Tensor: adj_mat = adj_mat.coalesce() indices = adj_mat.indices() values = adj_mat.values() - degrees_row = torch.zeros(adj_mat.shape[0]) + degrees_row = torch.zeros(adj_mat.shape[0], device=adj_mat.device) degrees_row = degrees_row.index_add(0, indices[0], values.to(degrees_row.dtype)) - degrees_col = torch.zeros(adj_mat.shape[1]) + degrees_col = torch.zeros(adj_mat.shape[1], device=adj_mat.device) degrees_col = degrees_col.index_add(0, indices[1], values.to(degrees_col.dtype)) values = values.to(degrees_row.dtype) / torch.sqrt(degrees_row[indices[0]] * degrees_col[indices[1]]) adj_mat = torch.sparse_coo_tensor(indices=indices, values=values, size=adj_mat.shape) diff --git a/src/icosagon/sampling.py b/src/icosagon/sampling.py index 28b143d..7c55944 100644 --- a/src/icosagon/sampling.py +++ b/src/icosagon/sampling.py @@ -18,8 +18,13 @@ def fixed_unigram_candidate_sampler( if isinstance(true_classes, torch.Tensor): true_classes = true_classes.detach().cpu().numpy() + + if isinstance(unigrams, torch.Tensor): + unigrams = unigrams.detach().cpu().numpy() + if len(true_classes.shape) != 2: raise ValueError('true_classes must be a 2D matrix with shape (num_samples, num_true)') + num_samples = true_classes.shape[0] unigrams = np.array(unigrams) if distortion != 1.: diff --git a/src/icosagon/trainprep.py b/src/icosagon/trainprep.py index b7074e1..c49300a 100644 --- a/src/icosagon/trainprep.py +++ b/src/icosagon/trainprep.py @@ -83,9 +83,11 @@ def train_val_test_split_edges(edges: torch.Tensor, def get_edges_and_degrees(adj_mat: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: if adj_mat.is_sparse: adj_mat = adj_mat.coalesce() - degrees = torch.zeros(adj_mat.shape[1], dtype=torch.int64) + degrees = torch.zeros(adj_mat.shape[1], dtype=torch.int64, + device=adj_mat.device) degrees = degrees.index_add(0, adj_mat.indices()[1], - torch.ones(adj_mat.indices().shape[1], dtype=torch.int64)) + torch.ones(adj_mat.indices().shape[1], dtype=torch.int64, + device=adj_mat.device)) edges_pos = adj_mat.indices().transpose(0, 1) else: degrees = adj_mat.sum(0) @@ -102,7 +104,7 @@ def prepare_adj_mat(adj_mat: torch.Tensor, edges_pos, degrees = get_edges_and_degrees(adj_mat) neg_neighbors = fixed_unigram_candidate_sampler( - edges_pos[:, 1].view(-1, 1), degrees, 0.75) + edges_pos[:, 1].view(-1, 1), degrees, 0.75).to(adj_mat.device) print(edges_pos.dtype) print(neg_neighbors.dtype) edges_neg = torch.cat((edges_pos[:, 0].view(-1, 1), neg_neighbors.view(-1, 1)), 1) @@ -111,7 +113,8 @@ def prepare_adj_mat(adj_mat: torch.Tensor, edges_neg = train_val_test_split_edges(edges_neg, ratios) adj_mat_train = torch.sparse_coo_tensor(indices = edges_pos.train.transpose(0, 1), - values=torch.ones(len(edges_pos.train)), size=adj_mat.shape, dtype=adj_mat.dtype) + values=torch.ones(len(edges_pos.train)), size=adj_mat.shape, dtype=adj_mat.dtype, + device=adj_mat.device) return adj_mat_train, edges_pos, edges_neg diff --git a/tests/icosagon/test_trainloop.py b/tests/icosagon/test_trainloop.py index 04956c6..0548462 100644 --- a/tests/icosagon/test_trainloop.py +++ b/tests/icosagon/test_trainloop.py @@ -4,6 +4,8 @@ from icosagon.trainprep import prepare_training, \ from icosagon.model import Model from icosagon.trainloop import TrainLoop import torch +import pytest +import pdb def test_train_loop_01(): @@ -37,3 +39,32 @@ def test_train_loop_02(): loop = TrainLoop(m) loop.run_epoch() + + +def test_train_loop_03(): + if torch.cuda.device_count() == 0: + pytest.skip('CUDA required for this test') + + adj_mat = torch.rand(10, 10).round() + dev = torch.device('cuda:0') + adj_mat = adj_mat.to(dev) + + d = Data() + d.add_node_type('Dummy', 10) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Rel', adj_mat) + + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) + # pdb.set_trace() + + m = Model(prep_d) + m = m.to(dev) + + print(list(m.parameters())) + + for prm in m.parameters(): + assert prm.device == dev + + loop = TrainLoop(m) + + loop.run_epoch() -- 2.26.2 From 3391e15da1f1863c5d692941881c815041fe8e0d Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 16 Jul 2020 20:21:47 +0200 Subject: [PATCH 140/227] Fix for sparse_coo_tensor(). --- src/icosagon/dropout.py | 3 ++- src/icosagon/normalize.py | 14 ++++++++++++-- tests/icosagon/test_trainloop.py | 1 + 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/icosagon/dropout.py b/src/icosagon/dropout.py index 95d0575..74bdd57 100644 --- a/src/icosagon/dropout.py +++ b/src/icosagon/dropout.py @@ -5,6 +5,7 @@ import torch +from .normalize import _sparse_coo_tensor def dropout_sparse(x, keep_prob): @@ -17,7 +18,7 @@ def dropout_sparse(x, keep_prob): n = torch.floor(n).to(torch.bool) i = i[:,n] v = v[n] - x = torch.sparse_coo_tensor(i, v, size=size) + x = _sparse_coo_tensor(i, v, size=size) return x * (1./keep_prob) diff --git a/src/icosagon/normalize.py b/src/icosagon/normalize.py index b41dfb1..ec09b3c 100644 --- a/src/icosagon/normalize.py +++ b/src/icosagon/normalize.py @@ -35,6 +35,16 @@ def _check_2d(adj_mat): raise ValueError('adj_mat must be a square matrix') +def _sparse_coo_tensor(indices, values, size): + ctor = { torch.float32: torch.sparse.FloatTensor, + torch.float32: torch.sparse.DoubleTensor, + torch.uint8: torch.sparse.ByteTensor, + torch.long: torch.sparse.LongTensor, + torch.int: torch.sparse.IntTensor, + torch.short: torch.sparse.ShortTensor }[values.dtype] + return ctor(indices, values, size) + + def add_eye_sparse(adj_mat: torch.Tensor) -> torch.Tensor: _check_tensor(adj_mat) _check_sparse(adj_mat) @@ -53,7 +63,7 @@ def add_eye_sparse(adj_mat: torch.Tensor) -> torch.Tensor: indices = torch.cat((indices, eye_indices), 1) values = torch.cat((values, eye_values), 0) - adj_mat = torch.sparse_coo_tensor(indices=indices, values=values, size=adj_mat.shape) + adj_mat = _sparse_coo_tensor(indices, values, adj_mat.shape) return adj_mat @@ -104,7 +114,7 @@ def norm_adj_mat_two_node_types_sparse(adj_mat: torch.Tensor) -> torch.Tensor: degrees_col = torch.zeros(adj_mat.shape[1], device=adj_mat.device) degrees_col = degrees_col.index_add(0, indices[1], values.to(degrees_col.dtype)) values = values.to(degrees_row.dtype) / torch.sqrt(degrees_row[indices[0]] * degrees_col[indices[1]]) - adj_mat = torch.sparse_coo_tensor(indices=indices, values=values, size=adj_mat.shape) + adj_mat = _sparse_coo_tensor(indices, values, adj_mat.shape) return adj_mat diff --git a/tests/icosagon/test_trainloop.py b/tests/icosagon/test_trainloop.py index 0548462..5271a06 100644 --- a/tests/icosagon/test_trainloop.py +++ b/tests/icosagon/test_trainloop.py @@ -42,6 +42,7 @@ def test_train_loop_02(): def test_train_loop_03(): + # pdb.set_trace() if torch.cuda.device_count() == 0: pytest.skip('CUDA required for this test') -- 2.26.2 From b9e7e395cb78c3ad0a0abc4d020c769f27612ac5 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 16 Jul 2020 20:23:26 +0200 Subject: [PATCH 141/227] Add shuffle param to TrainLoop. --- src/icosagon/trainloop.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/icosagon/trainloop.py b/src/icosagon/trainloop.py index 3392a40..098baba 100644 --- a/src/icosagon/trainloop.py +++ b/src/icosagon/trainloop.py @@ -8,10 +8,15 @@ from types import FunctionType class TrainLoop(object): - def __init__(self, model: Model, lr: float = 0.001, + def __init__( + self, + model: Model, + lr: float = 0.001, loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = \ torch.nn.functional.binary_cross_entropy_with_logits, - batch_size: int = 100, generator: torch.Generator = None) -> None: + batch_size: int = 100, + shuffle: bool = False, + generator: torch.Generator = None) -> None: if not isinstance(model, Model): raise TypeError('model must be an instance of Model') @@ -30,6 +35,7 @@ class TrainLoop(object): self.lr = lr self.loss = loss self.batch_size = batch_size + self.shuffle = shuffle self.generator = generator or torch.default_generator self.opt = None @@ -42,7 +48,7 @@ class TrainLoop(object): def run_epoch(self): batch = PredictionsBatch(self.model.prep_d, batch_size=self.batch_size, - generator=self.generator) + shuffle = self.shuffle, generator=self.generator) # pred = self.model(None) # n = len(list(iter(batch))) loss_sum = 0 -- 2.26.2 From 749ff00a754c3fb40f710dbbb23c92833ca005b7 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 16 Jul 2020 20:47:57 +0200 Subject: [PATCH 142/227] Shuffle. --- experiments/decagon_run/decagon_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiments/decagon_run/decagon_run.py b/experiments/decagon_run/decagon_run.py index 911caaa..74e1a03 100644 --- a/experiments/decagon_run/decagon_run.py +++ b/experiments/decagon_run/decagon_run.py @@ -118,7 +118,7 @@ def main(): # model = torch.nn.DataParallel(model, ['cuda:0', 'cuda:1']) _wrap(TrainLoop, 'build') _wrap(TrainLoop, 'run_epoch') - loop = TrainLoop(model, batch_size=1000000) + loop = TrainLoop(model, batch_size=512, shuffle=True) loop.run_epoch() -- 2.26.2 From a5b8701a0dde2373b75c4c91c27c46f0296fc15a Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 16 Jul 2020 21:32:43 +0200 Subject: [PATCH 143/227] Add some debug output for profiling, the bottleneck is in DecodeLayer but also comes generally from computing always all nodes. --- src/icosagon/convlayer.py | 3 +++ src/icosagon/declayer.py | 5 +++++ src/icosagon/trainloop.py | 15 +++++++++++++++ tests/icosagon/test_trainloop.py | 10 ++++++++++ 4 files changed, 33 insertions(+) diff --git a/src/icosagon/convlayer.py b/src/icosagon/convlayer.py index e98b55e..9b8e5ae 100644 --- a/src/icosagon/convlayer.py +++ b/src/icosagon/convlayer.py @@ -7,6 +7,7 @@ from typing import List, \ Callable from collections import defaultdict from dataclasses import dataclass +import time class Convolutions(torch.nn.Module): @@ -104,6 +105,7 @@ class DecagonLayer(torch.nn.Module): self.build_family(fam) def __call__(self, prev_layer_repr): + t = time.time() next_layer_repr = [ [] for _ in range(len(self.data.node_types)) ] n = len(self.data.node_types) @@ -120,4 +122,5 @@ class DecagonLayer(torch.nn.Module): next_layer_repr[node_type_row] = sum(next_layer_repr[node_type_row]) next_layer_repr[node_type_row] = self.layer_activation(next_layer_repr[node_type_row]) + print('DecagonLayer.forward() took', time.time() - t) return next_layer_repr diff --git a/src/icosagon/declayer.py b/src/icosagon/declayer.py index 8f6bff4..13f751d 100644 --- a/src/icosagon/declayer.py +++ b/src/icosagon/declayer.py @@ -16,6 +16,7 @@ from typing import Type, \ Tuple from .decode import DEDICOMDecoder from dataclasses import dataclass +import time @dataclass @@ -75,6 +76,7 @@ class DecodeLayer(torch.nn.Module): self.decoders.append(dec) def _get_tvt(self, r, edge_list_attr_names, row, column, k, last_layer_repr, dec): + start_time = time.time() pred = [] for p in edge_list_attr_names: tvt = [] @@ -86,9 +88,11 @@ class DecodeLayer(torch.nn.Module): tvt.append(dec(inputs_row, inputs_column, k)) tvt = TrainValTest(*tvt) pred.append(tvt) + print('DecodeLayer._get_tvt() took:', time.time() - start_time) return pred def forward(self, last_layer_repr: List[torch.Tensor]) -> List[List[torch.Tensor]]: + t = time.time() res = [] for i, fam in enumerate(self.data.relation_families): fam_pred = [] @@ -103,4 +107,5 @@ class DecodeLayer(torch.nn.Module): fam_pred = RelationFamilyPredictions(fam_pred) res.append(fam_pred) res = Predictions(res) + print('DecodeLayer.forward() took', time.time() - t) return res diff --git a/src/icosagon/trainloop.py b/src/icosagon/trainloop.py index 098baba..40cb122 100644 --- a/src/icosagon/trainloop.py +++ b/src/icosagon/trainloop.py @@ -5,6 +5,7 @@ from .batch import PredictionsBatch, \ gather_batch_indices from typing import Callable from types import FunctionType +import time class TrainLoop(object): @@ -54,9 +55,15 @@ class TrainLoop(object): loss_sum = 0 for i, indices in enumerate(batch): print('%.2f%% (%d/%d)' % (i * batch.batch_size * 100 / batch.total_edge_count, i * batch.batch_size, batch.total_edge_count)) + t = time.time() self.opt.zero_grad() + print('zero_grad() took:', time.time() - t) + t = time.time() pred = self.model(None) + print('model() took:', time.time() - t) + t = time.time() pred = flatten_predictions(pred) + print('flatten_predictions() took:', time.time() - t) # batch = PredictionsBatch(pred, batch_size=self.batch_size, shuffle=True) # seed = torch.rand(1).item() # rng_state = torch.get_rng_state() @@ -66,10 +73,18 @@ class TrainLoop(object): #for k in range(i): #_ = next(it) #(input, target) = next(it) + t = time.time() (input, target) = gather_batch_indices(pred, indices) + print('gather_batch_indices() took:', time.time() - t) + t = time.time() loss = self.loss(input, target) + print('loss() took:', time.time() - t) + t = time.time() loss.backward() + print('backward() took:', time.time() - t) + t = time.time() self.opt.step() + print('step() took:', time.time() - t) loss_sum += loss.detach().cpu().item() return loss_sum diff --git a/tests/icosagon/test_trainloop.py b/tests/icosagon/test_trainloop.py index 5271a06..be6273c 100644 --- a/tests/icosagon/test_trainloop.py +++ b/tests/icosagon/test_trainloop.py @@ -6,6 +6,7 @@ from icosagon.trainloop import TrainLoop import torch import pytest import pdb +import time def test_train_loop_01(): @@ -69,3 +70,12 @@ def test_train_loop_03(): loop = TrainLoop(m) loop.run_epoch() + + +def test_timing_01(): + adj_mat = (torch.rand(2000, 2000) < .001).to(torch.float32).to_sparse() + rep = torch.eye(2000).requires_grad_(True) + t = time.time() + for _ in range(1300): + _ = torch.sparse.mm(adj_mat, rep) + print('Elapsed:', time.time() - t) -- 2.26.2 From 82c0b5d5867596ae84f26494a5f75c0415f1fea5 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 16 Jul 2020 21:35:39 +0200 Subject: [PATCH 144/227] Update .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 9fd0d18..eb03072 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ __pycache__ .cache/ .coverage +/docs/icosagon/*.dot +/docs/icosagon/*.png +/experiments/decagon_run/profiler_results -- 2.26.2 From 13940671ccb2b0381b1ff03fb16f928aa226eb95 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 17 Jul 2020 11:55:38 +0200 Subject: [PATCH 145/227] Performance tests. --- docs/nodes-involved.svg | 438 ++++++++++++++++++ .../decagon_run_effcat/decagon_run_effcat.py | 131 ++++++ src/icosagon/compile.py | 28 ++ tests/icosagon/test_trainloop.py | 33 ++ 4 files changed, 630 insertions(+) create mode 100644 docs/nodes-involved.svg create mode 100644 experiments/decagon_run_effcat/decagon_run_effcat.py create mode 100644 src/icosagon/compile.py diff --git a/docs/nodes-involved.svg b/docs/nodes-involved.svg new file mode 100644 index 0000000..045d85e --- /dev/null +++ b/docs/nodes-involved.svg @@ -0,0 +1,438 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/experiments/decagon_run_effcat/decagon_run_effcat.py b/experiments/decagon_run_effcat/decagon_run_effcat.py new file mode 100644 index 0000000..afe65ec --- /dev/null +++ b/experiments/decagon_run_effcat/decagon_run_effcat.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 + +from icosagon.data import Data +from icosagon.trainprep import TrainValTest, \ + prepare_training +from icosagon.model import Model +from icosagon.trainloop import TrainLoop +import os +import pandas as pd +from bisect import bisect_left +import torch +import sys + + +def index(a, x): + i = bisect_left(a, x) + if i != len(a) and a[i] == x: + return i + raise ValueError + + +def load_data(dev): + path = '/pstore/data/data_science/ref/decagon' + df_combo = pd.read_csv(os.path.join(path, 'bio-decagon-combo.csv')) + df_effcat = pd.read_csv(os.path.join(path, 'bio-decagon-effectcategories.csv')) + df_mono = pd.read_csv(os.path.join(path, 'bio-decagon-mono.csv')) + df_ppi = pd.read_csv(os.path.join(path, 'bio-decagon-ppi.csv')) + df_tgtall = pd.read_csv(os.path.join(path, 'bio-decagon-targets-all.csv')) + df_tgt = pd.read_csv(os.path.join(path, 'bio-decagon-targets.csv')) + lst = [ 'df_combo', 'df_effcat', 'df_mono', 'df_ppi', 'df_tgtall', 'df_tgt' ] + for nam in lst: + print(f'len({nam}): {len(locals()[nam])}') + print(f'{nam}.columns: {locals()[nam].columns}') + + genes = set() + genes = genes.union(df_ppi['Gene 1']).union(df_ppi['Gene 2']) \ + .union(df_tgtall['Gene']).union(df_tgt['Gene']) + genes = sorted(genes) + print('len(genes):', len(genes)) + + drugs = set() + drugs = drugs.union(df_combo['STITCH 1']).union(df_combo['STITCH 2']) \ + .union(df_mono['STITCH']).union(df_tgtall['STITCH']).union(df_tgt['STITCH']) + drugs = sorted(drugs) + print('len(drugs):', len(drugs)) + + data = Data() + data.add_node_type('Gene', len(genes)) + data.add_node_type('Drug', len(drugs)) + + print('Preparing PPI...') + print('Indexing rows...') + rows = [index(genes, g) for g in df_ppi['Gene 1']] + print('Indexing cols...') + cols = [index(genes, g) for g in df_ppi['Gene 2']] + indices = list(zip(rows, cols)) + indices = torch.tensor(indices).transpose(0, 1) + values = torch.ones(len(rows)) + print('indices.shape:', indices.shape, 'values.shape:', values.shape) + adj_mat = torch.sparse_coo_tensor(indices, values, size=(len(genes),) * 2, + device=dev) + adj_mat = (adj_mat + adj_mat.transpose(0, 1)) / 2 + print('adj_mat created') + fam = data.add_relation_family('PPI', 0, 0, True) + rel = fam.add_relation_type('PPI', adj_mat) + print('OK') + + print('Preparing Drug-Gene (Target) edges...') + rows = [index(drugs, d) for d in df_tgtall['STITCH']] + cols = [index(genes, g) for g in df_tgtall['Gene']] + indices = list(zip(rows, cols)) + indices = torch.tensor(indices).transpose(0, 1) + values = torch.ones(len(rows)) + adj_mat = torch.sparse_coo_tensor(indices, values, size=(len(drugs), len(genes)), + device=dev) + fam = data.add_relation_family('Drug-Gene (Target)', 1, 0, True) + rel = fam.add_relation_type('Drug-Gene (Target)', adj_mat) + print('OK') + + df_combo_effcat = df_combo.merge(df_effcat, left_on='Polypharmacy Side Effect', right_on='Side Effect') + disease_classes = [] + + print('Preparing Drug-Drug (Side Effect) edges...') + fam = data.add_relation_family('Drug-Drug (Side Effect)', 1, 1, True) + print('# of side effects:', len(df_combo), 'unique:', len(df_combo['Polypharmacy Side Effect'].unique())) + for discls, df in df_combo_effcat.groupby('Disease Class'): + disease_classes.append(discls) + sys.stdout.write('.') # print(eff, '...') + sys.stdout.flush() + rows = [index(drugs, d) for d in df['STITCH 1']] + cols = [index(drugs, d) for d in df['STITCH 2']] + indices = list(zip(rows, cols)) + indices = torch.tensor(indices).transpose(0, 1) + values = torch.ones(len(rows)) + adj_mat = torch.sparse_coo_tensor(indices, values, size=(len(drugs), len(drugs)), + device=dev) + adj_mat = (adj_mat + adj_mat.transpose(0, 1)) / 2 + rel = fam.add_relation_type(df['Polypharmacy Side Effect'], adj_mat) + print() + print('len(disease_classes):', len(disease_classes)) + print('OK') + + return data + + +def _wrap(obj, method_name): + orig_fn = getattr(obj, method_name) + def fn(*args, **kwargs): + print(f'{method_name}() :: ENTER') + res = orig_fn(*args, **kwargs) + print(f'{method_name}() :: EXIT') + return res + setattr(obj, method_name, fn) + + +def main(): + dev = torch.device('cuda:0') + data = load_data(dev) + prep_d = prepare_training(data, TrainValTest(.8, .1, .1)) + _wrap(Model, 'build') + model = Model(prep_d) + model = model.to(dev) + # model = torch.nn.DataParallel(model, ['cuda:0', 'cuda:1']) + _wrap(TrainLoop, 'build') + _wrap(TrainLoop, 'run_epoch') + loop = TrainLoop(model, batch_size=512, shuffle=True) + loop.run_epoch() + + +if __name__ == '__main__': + main() diff --git a/src/icosagon/compile.py b/src/icosagon/compile.py new file mode 100644 index 0000000..a16c29d --- /dev/null +++ b/src/icosagon/compile.py @@ -0,0 +1,28 @@ +# +# The goal of this module is to make Icosagon more efficient. +# It takes the nice Icosagon model architecture and tries to +# formulate it in terms of batch matrix multiplications instead +# of using Python for loops. +# + +from .weights import init_glorot +from .input +import torch + + +class EncodeLayer(object): + def __init__(self, num_relation_types, input_dim, output_dim): + weights = [ init_glorot(input_dim, output_dim) \ + for _ in range(num_relation_types) ] + weights = torch.cat(weights) + + +class Compiler(object): + def __init__(self, data: Data, layer_dimensions: List[int] = [32, 64]) -> None: + self.data = data + self.layer_dimensions = layer_dimensions + self.build() + + def build(self) -> None: + for fam in data.relation_families: + init_glorot(in_channels, out_channels) diff --git a/tests/icosagon/test_trainloop.py b/tests/icosagon/test_trainloop.py index be6273c..accbea9 100644 --- a/tests/icosagon/test_trainloop.py +++ b/tests/icosagon/test_trainloop.py @@ -79,3 +79,36 @@ def test_timing_01(): for _ in range(1300): _ = torch.sparse.mm(adj_mat, rep) print('Elapsed:', time.time() - t) + + +def test_timing_02(): + adj_mat = (torch.rand(2000, 2000) < .001).to(torch.float32) + adj_mat_batch = [adj_mat.view(1, 2000, 2000)] * 1300 + adj_mat_batch = torch.cat(adj_mat_batch) + rep = torch.eye(2000).requires_grad_(True) + t = time.time() + res = torch.matmul(adj_mat_batch, rep) + print('Elapsed:', time.time() - t) + print('res.shape:', res.shape) + + +def test_timing_03(): + adj_mat = (torch.rand(2000, 2000) < .001).to(torch.float32) + adj_mat_batch = [adj_mat.view(1, 2000, 2000).to_sparse()] * 1300 + adj_mat_batch = torch.cat(adj_mat_batch) + rep = torch.eye(2000).requires_grad_(True) + rep_batch = [rep.view(1, 2000, 2000)] * 1300 + rep_batch = torch.cat(rep_batch) + t = time.time() + with pytest.raises(RuntimeError): + _ = torch.bmm(adj_mat_batch, rep) + print('Elapsed:', time.time() - t) + + +def test_timing_04(): + adj_mat = (torch.rand(2000, 2000) < .0001).to(torch.float32).to_sparse() + rep = torch.eye(2000).requires_grad_(True) + t = time.time() + for _ in range(1300): + _ = torch.sparse.mm(adj_mat, rep) + print('Elapsed:', time.time() - t) -- 2.26.2 From 431c871af76c7fd4c20c20f92ce295e8dd513102 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 17 Jul 2020 12:58:39 +0200 Subject: [PATCH 146/227] Add test_timing_05(). --- .gitignore | 1 + tests/icosagon/test_trainloop.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/.gitignore b/.gitignore index eb03072..9ec7cab 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ __pycache__ /docs/icosagon/*.dot /docs/icosagon/*.png /experiments/decagon_run/profiler_results +/experiments/decagon_run_effcat/profiler_results diff --git a/tests/icosagon/test_trainloop.py b/tests/icosagon/test_trainloop.py index accbea9..c052dba 100644 --- a/tests/icosagon/test_trainloop.py +++ b/tests/icosagon/test_trainloop.py @@ -112,3 +112,16 @@ def test_timing_04(): for _ in range(1300): _ = torch.sparse.mm(adj_mat, rep) print('Elapsed:', time.time() - t) + + +def test_timing_05(): + if torch.cuda.device_count() == 0: + pytest.skip('Test requires CUDA') + dev = torch.device('cuda:0') + adj_mat = (torch.rand(2000, 2000) < .001).to(torch.float32).to_sparse().to(dev) + rep = torch.eye(2000).requires_grad_(True).to(dev) + t = time.time() + for _ in range(1300): + _ = torch.sparse.mm(adj_mat, rep) + torch.cuda.synchronize() + print('Elapsed:', time.time() - t) -- 2.26.2 From cd4c34ea7d39c0f3969c7894545f73cc9d178961 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 17 Jul 2020 13:20:06 +0200 Subject: [PATCH 147/227] Fix for non-leaf .grad warning. --- src/icosagon/input.py | 6 +++--- tests/icosagon/test_trainloop.py | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/icosagon/input.py b/src/icosagon/input.py index 8c686b1..3bf5824 100644 --- a/src/icosagon/input.py +++ b/src/icosagon/input.py @@ -60,11 +60,11 @@ class OneHotInputLayer(torch.nn.Module): self.build() def build(self) -> None: - self.node_reps = [] + self.node_reps = torch.nn.ParameterList() for i, nt in enumerate(self.data.node_types): reps = torch.eye(nt.count).to_sparse() - reps = torch.nn.Parameter(reps) - self.register_parameter('node_reps[%d]' % i, reps) + reps = torch.nn.Parameter(reps, requires_grad=False) + # self.register_parameter('node_reps[%d]' % i, reps) self.node_reps.append(reps) def forward(self, x) -> List[torch.nn.Parameter]: diff --git a/tests/icosagon/test_trainloop.py b/tests/icosagon/test_trainloop.py index c052dba..ca77edd 100644 --- a/tests/icosagon/test_trainloop.py +++ b/tests/icosagon/test_trainloop.py @@ -37,10 +37,16 @@ def test_train_loop_02(): m = Model(prep_d) + for prm in m.parameters(): + print(prm.shape, prm.is_leaf, prm.requires_grad) + loop = TrainLoop(m) loop.run_epoch() + for prm in m.parameters(): + print(prm.shape, prm.is_leaf, prm.requires_grad) + def test_train_loop_03(): # pdb.set_trace() -- 2.26.2 From 94314d2e528b68fb6ade182e1859b41385fac839 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 17 Jul 2020 13:26:27 +0200 Subject: [PATCH 148/227] Fix in test_decagon_layer_04(). --- tests/icosagon/test_convlayer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/icosagon/test_convlayer.py b/tests/icosagon/test_convlayer.py index 636301f..a6daa99 100644 --- a/tests/icosagon/test_convlayer.py +++ b/tests/icosagon/test_convlayer.py @@ -124,7 +124,7 @@ def test_decagon_layer_04(): seq = torch.nn.Sequential(in_layer, d_layer) out_d_layer = seq(None) - out_multi_dgca = multi_dgca(in_layer(None)) + out_multi_dgca = multi_dgca(list(in_layer(None))) assert isinstance(out_d_layer, list) assert len(out_d_layer) == 1 -- 2.26.2 From 4a9ddd540fb10485d5d63c0357dcb9f50a890e90 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 17 Jul 2020 13:27:30 +0200 Subject: [PATCH 149/227] Fix in test_decode_layer_05(). --- tests/icosagon/test_declayer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/icosagon/test_declayer.py b/tests/icosagon/test_declayer.py index 732fd45..014d304 100644 --- a/tests/icosagon/test_declayer.py +++ b/tests/icosagon/test_declayer.py @@ -166,7 +166,7 @@ def test_decode_layer_05(): # print(rel_pred.edges_neg.train) repr_in = in_layer(None) - assert isinstance(repr_in, list) + assert isinstance(repr_in, torch.nn.ParameterList) assert len(repr_in) == 1 assert isinstance(repr_in[0], torch.Tensor) assert torch.all(repr_in[0].to_dense() == torch.eye(10)) -- 2.26.2 From 1ee89509ee4281317df675926faf19ce58c27588 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 17 Jul 2020 13:59:30 +0200 Subject: [PATCH 150/227] Fixes for bool sparse tensors. --- src/icosagon/normalize.py | 3 ++- tests/icosagon/test_normalize.py | 4 ++-- tests/icosagon/test_trainprep.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/icosagon/normalize.py b/src/icosagon/normalize.py index ec09b3c..e13fb05 100644 --- a/src/icosagon/normalize.py +++ b/src/icosagon/normalize.py @@ -41,7 +41,8 @@ def _sparse_coo_tensor(indices, values, size): torch.uint8: torch.sparse.ByteTensor, torch.long: torch.sparse.LongTensor, torch.int: torch.sparse.IntTensor, - torch.short: torch.sparse.ShortTensor }[values.dtype] + torch.short: torch.sparse.ShortTensor, + torch.bool: torch.sparse.ByteTensor }[values.dtype] return ctor(indices, values, size) diff --git a/tests/icosagon/test_normalize.py b/tests/icosagon/test_normalize.py index e4b0fef..f3de835 100644 --- a/tests/icosagon/test_normalize.py +++ b/tests/icosagon/test_normalize.py @@ -46,14 +46,14 @@ def test_add_eye_sparse_04(): def test_norm_adj_mat_one_node_type_sparse_01(): adj_mat = torch.rand((10, 10)) - adj_mat = (adj_mat > .5) + adj_mat = (adj_mat > .5).to(torch.float32) adj_mat = adj_mat.to_sparse() _ = norm_adj_mat_one_node_type_sparse(adj_mat) def test_norm_adj_mat_one_node_type_sparse_02(): adj_mat_dense = torch.rand((10, 10)) - adj_mat_dense = (adj_mat_dense > .5) + adj_mat_dense = (adj_mat_dense > .5).to(torch.float32) adj_mat_sparse = adj_mat_dense.to_sparse() adj_mat_sparse = norm_adj_mat_one_node_type_sparse(adj_mat_sparse) adj_mat_dense = norm_adj_mat_one_node_type_dense(adj_mat_dense) diff --git a/tests/icosagon/test_trainprep.py b/tests/icosagon/test_trainprep.py index b49646b..5ec86a7 100644 --- a/tests/icosagon/test_trainprep.py +++ b/tests/icosagon/test_trainprep.py @@ -108,7 +108,7 @@ def test_prepare_adj_mat_02(): def test_prepare_relation_type_01(): - adj_mat = (torch.rand((10, 10)) > .5) + adj_mat = (torch.rand((10, 10)) > .5).to(torch.float32) r = RelationType('Test', 0, 0, adj_mat, True) ratios = TrainValTest(.8, .1, .1) _ = prepare_relation_type(r, ratios, False) -- 2.26.2 From 6cf317595081f117622db480c933b20d06018c2f Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 17 Jul 2020 14:17:01 +0200 Subject: [PATCH 151/227] Remove an extra newline. --- src/icosagon/convolve.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/icosagon/convolve.py b/src/icosagon/convolve.py index 09bd32a..09b6eca 100644 --- a/src/icosagon/convolve.py +++ b/src/icosagon/convolve.py @@ -20,7 +20,6 @@ class GraphConv(torch.nn.Module): self.weight = torch.nn.Parameter(init_glorot(in_channels, out_channels)) self.adjacency_matrix = adjacency_matrix - def forward(self, x: torch.Tensor) -> torch.Tensor: x = torch.sparse.mm(x, self.weight) \ if x.is_sparse \ -- 2.26.2 From 47645ef409f4ae75f41b8fa1452ac4d4754ab75c Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 17 Jul 2020 18:46:49 +0200 Subject: [PATCH 152/227] Add Citing note. --- README.md | 7 ++++++- experiments/decagon_run/decagon_run.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ab1c38f..a1c01f7 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,14 @@ settings. Decagon-PyTorch is a PyTorch reimplementation of the algorithm. +## Citing + +If you use this code in your research please cite this repository as: + +Adaszewski S. (2020) https://code.adared.ch/sadaszewski/decagon-pytorch + ## References 1. Zitnik, M., Agrawal, M., & Leskovec, J. (2018). [Modeling polypharmacy side effects with graph convolutional networks](https://academic.oup.com/bioinformatics/article/34/13/i457/5045770) Bioinformatics, 34(13), i457-i466. - diff --git a/experiments/decagon_run/decagon_run.py b/experiments/decagon_run/decagon_run.py index 74e1a03..4093e08 100644 --- a/experiments/decagon_run/decagon_run.py +++ b/experiments/decagon_run/decagon_run.py @@ -109,7 +109,7 @@ def _wrap(obj, method_name): def main(): - dev = torch.device('cuda:0') + dev = torch.device('cpu') data = load_data(dev) prep_d = prepare_training(data, TrainValTest(.8, .1, .1)) _wrap(Model, 'build') -- 2.26.2 From fc8f9726afda98aaf0242688ba9830510b2f777b Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 21 Jul 2020 09:08:20 +0200 Subject: [PATCH 153/227] Remove unnecessary .detach() call in dropout_dense(). --- src/icosagon/dropout.py | 3 +- tests/icosagon/test_trainloop.py | 53 +++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/src/icosagon/dropout.py b/src/icosagon/dropout.py index 74bdd57..63cfb58 100644 --- a/src/icosagon/dropout.py +++ b/src/icosagon/dropout.py @@ -24,7 +24,8 @@ def dropout_sparse(x, keep_prob): def dropout_dense(x, keep_prob): - x = x.clone().detach() + # print('dropout_dense()') + x = x.clone() i = torch.nonzero(x) n = keep_prob + torch.rand(len(i)) diff --git a/tests/icosagon/test_trainloop.py b/tests/icosagon/test_trainloop.py index ca77edd..192cdf9 100644 --- a/tests/icosagon/test_trainloop.py +++ b/tests/icosagon/test_trainloop.py @@ -1,4 +1,5 @@ -from icosagon.data import Data +from icosagon.data import Data, \ + _equal from icosagon.trainprep import prepare_training, \ TrainValTest from icosagon.model import Model @@ -78,6 +79,56 @@ def test_train_loop_03(): loop.run_epoch() +def test_train_loop_04(): + adj_mat = torch.rand(10, 10).round() + + d = Data() + d.add_node_type('Dummy', 10) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Rel', adj_mat) + + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) + + m = Model(prep_d) + + old_values = [] + for prm in m.parameters(): + old_values.append(prm.clone().detach()) + + loop = TrainLoop(m) + + loop.run_epoch() + + for i, prm in enumerate(m.parameters()): + assert not prm.requires_grad or \ + not torch.all(_equal(prm, old_values[i])) + + +def test_train_loop_05(): + adj_mat = torch.rand(10, 10).round().to_sparse() + + d = Data() + d.add_node_type('Dummy', 10) + fam = d.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Rel', adj_mat) + + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) + + m = Model(prep_d) + + old_values = [] + for prm in m.parameters(): + old_values.append(prm.clone().detach()) + + loop = TrainLoop(m) + + loop.run_epoch() + + for i, prm in enumerate(m.parameters()): + assert not prm.requires_grad or \ + not torch.all(_equal(prm, old_values[i])) + + def test_timing_01(): adj_mat = (torch.rand(2000, 2000) < .001).to(torch.float32).to_sparse() rep = torch.eye(2000).requires_grad_(True) -- 2.26.2 From e74eb25a31845e3d59b6ab3d2c728a6d05350172 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 21 Jul 2020 10:57:50 +0200 Subject: [PATCH 154/227] Add bulkdec. --- src/icosagon/bulkdec.py | 119 +++++++++++++++++++++++++++++++++ tests/icosagon/test_bulkdec.py | 113 +++++++++++++++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 src/icosagon/bulkdec.py create mode 100644 tests/icosagon/test_bulkdec.py diff --git a/src/icosagon/bulkdec.py b/src/icosagon/bulkdec.py new file mode 100644 index 0000000..ea6dba8 --- /dev/null +++ b/src/icosagon/bulkdec.py @@ -0,0 +1,119 @@ +from icosagon.data import Data +from icosagon.trainprep import PreparedData +from icosagon.decode import DEDICOMDecoder, \ + DistMultDecoder, \ + BilinearDecoder, \ + InnerProductDecoder +from icosagon.dropout import dropout +import torch +from typing import List, \ + Callable, \ + Union + + +''' +Let's say that I have dense latent representations row and col. +Then let's take relation matrix rel in a list of relations REL. +A single computation currenty looks like this: +(((row * rel) * glob) * rel) * col + +Shouldn't then this basically work: + +prod1 = torch.matmul(row, REL) +prod2 = torch.matmul(prod1, glob) +prod3 = torch.matmul(prod2, REL) +res = torch.matmul(prod3, col) +res = activation(res) + +res should then have shape: (num_relations, num_rows, num_columns) +''' + + +def convert_decoder(dec): + if isinstance(dec, DEDICOMDecoder): + global_interaction = dec.global_interaction + local_variation = map(torch.diag, dec.local_variation) + elif isinstance(dec, DistMultDecoder): + global_interaction = torch.eye(dec.input_dim, dec.input_dim) + local_variation = map(torch.diag, dec.relation) + elif isinstance(dec, BilinearDecoder): + global_interaction = torch.eye(dec.input_dim, dec.input_dim) + local_variation = dec.relation + elif isinstance(dec, InnerProductDecoder): + global_interaction = torch.eye(dec.input_dim, dec.input_dim) + local_variation = torch.eye(dec.input_dim, dec.input_dim) + local_variation = [ local_variation ] * dec.num_relation_types + else: + raise TypeError('Unknown decoder type in covert_decoder()') + + if not isinstance(local_variation, torch.Tensor): + local_variation = map(lambda a: a.view(1, *a.shape), local_variation) + local_variation = torch.cat(list(local_variation)) + + return (global_interaction, local_variation) + + +class BulkDecodeLayer(torch.nn.Module): + def __init__(self, + input_dim: List[int], + data: Union[Data, PreparedData], + keep_prob: float = 1., + activation: Callable[[torch.Tensor], torch.Tensor] = torch.sigmoid, + **kwargs) -> None: + + super().__init__(**kwargs) + + self._check_params(input_dim, data) + + self.input_dim = input_dim[0] + self.data = data + self.keep_prob = keep_prob + self.activation = activation + + self.decoders = None + self.global_interaction = None + self.local_variation = None + self.build() + + def build(self) -> None: + self.decoders = torch.nn.ModuleList() + self.global_interaction = torch.nn.ParameterList() + self.local_variation = torch.nn.ParameterList() + for fam in self.data.relation_families: + dec = fam.decoder_class(self.input_dim, + len(fam.relation_types), + self.keep_prob, + self.activation) + self.decoders.append(dec) + global_interaction, local_variation = convert_decoder(dec) + self.global_interaction.append(torch.nn.Parameter(global_interaction)) + self.local_variation.append(torch.nn.Parameter(local_variation)) + + def forward(self, last_layer_repr: List[torch.Tensor]) -> List[torch.Tensor]: + res = [] + for i, fam in enumerate(self.data.relation_families): + repr_row = last_layer_repr[fam.node_type_row] + repr_column = last_layer_repr[fam.node_type_column] + repr_row = dropout(repr_row, keep_prob=self.keep_prob) + repr_column = dropout(repr_column, keep_prob=self.keep_prob) + prod_1 = torch.matmul(repr_row, self.local_variation[i]) + print(f'local_variation[{i}].shape: {self.local_variation[i].shape}') + prod_2 = torch.matmul(prod_1, self.global_interaction[i]) + prod_3 = torch.matmul(prod_2, self.local_variation[i]) + pred = torch.matmul(prod_3, repr_column.transpose(0, 1)) + res.append(pred) + return res + + @staticmethod + def _check_params(input_dim, data): + if not isinstance(input_dim, list): + raise TypeError('input_dim must be a list') + + if len(input_dim) != len(data.node_types): + raise ValueError('input_dim must have length equal to num_node_types') + + if not all([ a == input_dim[0] for a in input_dim ]): + raise ValueError('All elements of input_dim must have the same value') + + if not isinstance(data, Data) and not isinstance(data, PreparedData): + raise TypeError('data must be an instance of Data or PreparedData') diff --git a/tests/icosagon/test_bulkdec.py b/tests/icosagon/test_bulkdec.py new file mode 100644 index 0000000..c1e4670 --- /dev/null +++ b/tests/icosagon/test_bulkdec.py @@ -0,0 +1,113 @@ +from icosagon.data import Data +from icosagon.bulkdec import BulkDecodeLayer +from icosagon.input import OneHotInputLayer +from icosagon.convlayer import DecagonLayer +import torch + + +def test_bulk_decode_layer_01(): + data = Data() + data.add_node_type('Dummy', 100) + fam = data.add_relation_family('Dummy-Dummy', 0, 0, False) + fam.add_relation_type('Dummy Relation 1', + torch.rand((100, 100), dtype=torch.float32).round().to_sparse()) + + in_layer = OneHotInputLayer(data) + d_layer = DecagonLayer(in_layer.output_dim, 32, data) + dec_layer = BulkDecodeLayer(input_dim=d_layer.output_dim, data=data, + keep_prob=1., activation=lambda x: x) + seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) + + pred = seq(None) + + assert isinstance(pred, list) + assert len(pred) == len(data.relation_families) + assert isinstance(pred[0], torch.Tensor) + assert len(pred[0].shape) == 3 + assert len(pred[0]) == len(data.relation_families[0].relation_types) + assert pred[0].shape[1] == data.node_types[0].count + assert pred[0].shape[2] == data.node_types[0].count + + +def test_bulk_decode_layer_02(): + data = Data() + data.add_node_type('Foo', 100) + data.add_node_type('Bar', 50) + fam = data.add_relation_family('Foo-Bar', 0, 1, False) + fam.add_relation_type('Foobar Relation 1', + torch.rand((100, 50), dtype=torch.float32).round().to_sparse(), + torch.rand((50, 100), dtype=torch.float32).round().to_sparse()) + + in_layer = OneHotInputLayer(data) + d_layer = DecagonLayer(in_layer.output_dim, 32, data) + dec_layer = BulkDecodeLayer(input_dim=d_layer.output_dim, data=data, + keep_prob=1., activation=lambda x: x) + seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) + + pred = seq(None) + + assert isinstance(pred, list) + assert len(pred) == len(data.relation_families) + assert isinstance(pred[0], torch.Tensor) + assert len(pred[0].shape) == 3 + assert len(pred[0]) == len(data.relation_families[0].relation_types) + assert pred[0].shape[1] == data.node_types[0].count + assert pred[0].shape[2] == data.node_types[1].count + + +def test_bulk_decode_layer_03(): + data = Data() + data.add_node_type('Foo', 100) + data.add_node_type('Bar', 50) + fam = data.add_relation_family('Foo-Bar', 0, 1, False) + fam.add_relation_type('Foobar Relation 1', + torch.rand((100, 50), dtype=torch.float32).round().to_sparse(), + torch.rand((50, 100), dtype=torch.float32).round().to_sparse()) + fam.add_relation_type('Foobar Relation 2', + torch.rand((100, 50), dtype=torch.float32).round().to_sparse(), + torch.rand((50, 100), dtype=torch.float32).round().to_sparse()) + + in_layer = OneHotInputLayer(data) + d_layer = DecagonLayer(in_layer.output_dim, 32, data) + dec_layer = BulkDecodeLayer(input_dim=d_layer.output_dim, data=data, + keep_prob=1., activation=lambda x: x) + seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) + + pred = seq(None) + + assert isinstance(pred, list) + assert len(pred) == len(data.relation_families) + assert isinstance(pred[0], torch.Tensor) + assert len(pred[0].shape) == 3 + assert len(pred[0]) == len(data.relation_families[0].relation_types) + assert pred[0].shape[1] == data.node_types[0].count + assert pred[0].shape[2] == data.node_types[1].count + + +def test_bulk_decode_layer_03_big(): + data = Data() + data.add_node_type('Foo', 2000) + data.add_node_type('Bar', 2100) + fam = data.add_relation_family('Foo-Bar', 0, 1, False) + fam.add_relation_type('Foobar Relation 1', + torch.rand((2000, 2100), dtype=torch.float32).round().to_sparse(), + torch.rand((2100, 2000), dtype=torch.float32).round().to_sparse()) + fam.add_relation_type('Foobar Relation 2', + torch.rand((2000, 2100), dtype=torch.float32).round().to_sparse(), + torch.rand((2100, 2000), dtype=torch.float32).round().to_sparse()) + + in_layer = OneHotInputLayer(data) + d_layer = DecagonLayer(in_layer.output_dim, 32, data) + dec_layer = BulkDecodeLayer(input_dim=d_layer.output_dim, data=data, + keep_prob=1., activation=lambda x: x) + seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) + + pred = seq(None) + + assert isinstance(pred, list) + assert len(pred) == len(data.relation_families) + assert isinstance(pred[0], torch.Tensor) + assert len(pred[0].shape) == 3 + assert len(pred[0]) == len(data.relation_families[0].relation_types) + assert pred[0].shape[1] == data.node_types[0].count + assert pred[0].shape[2] == data.node_types[1].count -- 2.26.2 From cef1d1110ab1e24bf8830c3fdcedc3a029769463 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 21 Jul 2020 12:51:51 +0200 Subject: [PATCH 155/227] Add more tests in test_bulkdec. --- tests/icosagon/test_bulkdec.py | 127 +++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/tests/icosagon/test_bulkdec.py b/tests/icosagon/test_bulkdec.py index c1e4670..b4ed3d6 100644 --- a/tests/icosagon/test_bulkdec.py +++ b/tests/icosagon/test_bulkdec.py @@ -3,6 +3,9 @@ from icosagon.bulkdec import BulkDecodeLayer from icosagon.input import OneHotInputLayer from icosagon.convlayer import DecagonLayer import torch +import pytest +import time +import sys def test_bulk_decode_layer_01(): @@ -111,3 +114,127 @@ def test_bulk_decode_layer_03_big(): assert len(pred[0]) == len(data.relation_families[0].relation_types) assert pred[0].shape[1] == data.node_types[0].count assert pred[0].shape[2] == data.node_types[1].count + + +def test_bulk_decode_layer_03_huge_gpu(): + if torch.cuda.device_count() == 0: + pytest.skip('test_bulk_decode_layer_03_huge_gpu() requires CUDA support') + + device = torch.device('cuda:0') + data = Data() + data.add_node_type('Foo', 20000) + data.add_node_type('Bar', 21000) + fam = data.add_relation_family('Foo-Bar', 0, 1, False) + print('Adding Foobar Relation 1...') + fam.add_relation_type('Foobar Relation 1', + torch.rand((20000, 21000), dtype=torch.float32).round().to_sparse().to(device), + torch.rand((21000, 20000), dtype=torch.float32).round().to_sparse().to(device)) + print('Adding Foobar Relation 2...') + fam.add_relation_type('Foobar Relation 2', + torch.rand((20000, 21000), dtype=torch.float32).round().to_sparse().to(device), + torch.rand((21000, 20000), dtype=torch.float32).round().to_sparse().to(device)) + + in_layer = OneHotInputLayer(data) + d_layer = DecagonLayer(in_layer.output_dim, 32, data) + dec_layer = BulkDecodeLayer(input_dim=d_layer.output_dim, data=data, + keep_prob=1., activation=lambda x: x) + seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) + seq = seq.to(device) + + print('Starting forward pass...') + t = time.time() + pred = seq(None) + print('Elapsed:', time.time() - t) + + assert isinstance(pred, list) + assert len(pred) == len(data.relation_families) + assert isinstance(pred[0], torch.Tensor) + assert len(pred[0].shape) == 3 + assert len(pred[0]) == len(data.relation_families[0].relation_types) + assert pred[0].shape[1] == data.node_types[0].count + assert pred[0].shape[2] == data.node_types[1].count + + +def test_bulk_decode_layer_04_huge_multirel_gpu(): + if torch.cuda.device_count() == 0: + pytest.skip('test_bulk_decode_layer_04_huge_multirel_gpu() requires CUDA support') + + if torch.cuda.get_device_properties(0).total_memory < 64000000000: + pytest.skip('test_bulk_decode_layer_04_huge_multirel_gpu() requires GPU with 64GB of memory') + + device = torch.device('cuda:0') + data = Data() + data.add_node_type('Foo', 20000) + data.add_node_type('Bar', 21000) + fam = data.add_relation_family('Foo-Bar', 0, 1, False) + print('Generating adj_mat ...') + adj_mat = torch.rand((20000, 21000), dtype=torch.float32).round().to_sparse().to(device) + print('Generating adj_mat_back ...') + adj_mat_back = torch.rand((21000, 20000), dtype=torch.float32).round().to_sparse().to(device) + print('Adding relations ...') + for i in range(1300): + sys.stdout.write('.') + sys.stdout.flush() + fam.add_relation_type(f'Foobar Relation {i}', adj_mat, adj_mat_back) + print() + + in_layer = OneHotInputLayer(data) + d_layer = DecagonLayer(in_layer.output_dim, 32, data) + dec_layer = BulkDecodeLayer(input_dim=d_layer.output_dim, data=data, + keep_prob=1., activation=lambda x: x) + seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) + seq = seq.to(device) + + print('Starting forward pass...') + t = time.time() + pred = seq(None) + print('Elapsed:', time.time() - t) + + assert isinstance(pred, list) + assert len(pred) == len(data.relation_families) + assert isinstance(pred[0], torch.Tensor) + assert len(pred[0].shape) == 3 + assert len(pred[0]) == len(data.relation_families[0].relation_types) + assert pred[0].shape[1] == data.node_types[0].count + assert pred[0].shape[2] == data.node_types[1].count + + +def test_bulk_decode_layer_04_big_multirel_gpu(): + if torch.cuda.device_count() == 0: + pytest.skip('test_bulk_decode_layer_04_big_multirel_gpu() requires CUDA support') + + device = torch.device('cuda:0') + data = Data() + data.add_node_type('Foo', 2000) + data.add_node_type('Bar', 2100) + fam = data.add_relation_family('Foo-Bar', 0, 1, False) + print('Generating adj_mat ...') + adj_mat = torch.rand((2000, 2100), dtype=torch.float32).round().to_sparse().to(device) + print('Generating adj_mat_back ...') + adj_mat_back = torch.rand((2100, 2000), dtype=torch.float32).round().to_sparse().to(device) + print('Adding relations ...') + for i in range(1300): + sys.stdout.write('.') + sys.stdout.flush() + fam.add_relation_type(f'Foobar Relation {i}', adj_mat, adj_mat_back) + print() + + in_layer = OneHotInputLayer(data) + d_layer = DecagonLayer(in_layer.output_dim, 32, data) + dec_layer = BulkDecodeLayer(input_dim=d_layer.output_dim, data=data, + keep_prob=1., activation=lambda x: x) + seq = torch.nn.Sequential(in_layer, d_layer, dec_layer) + seq = seq.to(device) + + print('Starting forward pass...') + t = time.time() + pred = seq(None) + print('Elapsed:', time.time() - t) + + assert isinstance(pred, list) + assert len(pred) == len(data.relation_families) + assert isinstance(pred[0], torch.Tensor) + assert len(pred[0].shape) == 3 + assert len(pred[0]) == len(data.relation_families[0].relation_types) + assert pred[0].shape[1] == data.node_types[0].count + assert pred[0].shape[2] == data.node_types[1].count -- 2.26.2 From c210be41b3a615162c83d16d2128307d2a23066a Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 21 Jul 2020 14:14:07 +0200 Subject: [PATCH 156/227] Add databatch. --- src/icosagon/databatch.py | 91 ++++++++++++++++++++++++++++++++ tests/icosagon/test_databatch.py | 81 ++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 src/icosagon/databatch.py create mode 100644 tests/icosagon/test_databatch.py diff --git a/src/icosagon/databatch.py b/src/icosagon/databatch.py new file mode 100644 index 0000000..b7c4a8f --- /dev/null +++ b/src/icosagon/databatch.py @@ -0,0 +1,91 @@ +from icosagon.trainprep import PreparedData, \ + PreparedRelationFamily, \ + PreparedRelationType, \ + _empty_edge_list_tvt + + +class BatchedData(PreparedData): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +def batched_data_skeleton(data: PreparedData) -> BatchedData: + if not isinstance(data, PreparedData): + raise TypeError('data must be an instance of PreparedData') + + fam_skels = [] + for fam in data.relation_families: + rel_types_skel = [] + for rel in fam.relation_types: + rel_skel = PreparedRelationType(rel.name, + rel.node_type_row, rel.node_type_column, + rel.adjacency_matrix, rel.adjacency_matrix_backward, + _empty_edge_list_tvt(), _empty_edge_list_tvt(), + _empty_edge_list_tvt(), _empty_edge_list_tvt()) + rel_types_skel.append(rel_skel) + fam_skels.append(PreparedRelationFamily(fam.data, fam.name, + fam.node_type_row, fam.node_type_column, + fam.is_symmetric, fam.decoder_class, + rel_types_skel)) + return BatchedData(data.node_types, fam_skels) + + +class DataBatcher(object): + def __init__(self, data: PreparedData, batch_size: int) -> None: + self._check_params(data, batch_size) + + self.data = data + self.batch_size = batch_size + + # def batched_data_iter(self, fam_idx: int, rel_idx: int, + # part_type: str) -> BatchedData: + # + # rel = self.data.relation_families[fam_idx].relation_types[rel_idx] + # + # edges = getattr(rel.edges_pos, part_type) + # for m in range(0, len(edges), self.batch_size): + # batched_data = batched_data_skeleton(self.data) + # setattr(batched_data.relation_families[fam_idx].relation_types[rel_idx].edges_pos, + # part_type, edges[m : m + self.batch_size]) + # yield batched_data + # + # edges = getattr(rel.edges_neg, part_type) + # for m in range(0, len(edges), self.batch_size): + # batched_data = batched_data_skeleton(self.data) + # setattr(batched_data.relation_families[fam_idx].relation_types[rel_idx].edges_neg, + # part_type, edges[m : m + self.batch_size]) + # yield batched_data + # + # edges = getattr(rel.edges_pos_back, part_type) + # for m in range(0, len(edges), self.batch_size): + # batched_data = batched_data_skeleton(self.data) + # setattr(batched_data.relation_families[i].relation_types[k].edges_pos_back, + # part_type, edges[m : m + self.batch_size]) + # yield batched_data + # + # edges = getattr(rel.edges_neg_back, part_type) + # for m in range(0, len(), self.batch_size): + # batched_data = batched_data_skeleton(self.data) + # setattr(batched_data.relation_families[i].relation_types[k].edges_neg_back, + # edges[m : m + self.batch_size]) + # yield batched_data + + def __iter__(self) -> BatchedData: + for i, fam in enumerate(self.data.relation_families): + for k, rel in enumerate(fam.relation_types): + for edge_type in ['edges_pos', 'edges_neg', 'edges_back_pos', 'edges_back_neg']: + for part_type in ['train', 'val', 'test']: + edges = getattr(getattr(rel, edge_type), part_type) + for m in range(0, len(edges), self.batch_size): + batched_data = batched_data_skeleton(self.data) + setattr(getattr(batched_data.relation_families[i].relation_types[k], + edge_type), part_type, edges[m : m + self.batch_size]) + yield batched_data + + @staticmethod + def _check_params(data, batch_size): + if not isinstance(data, PreparedData): + raise TypeError('data must be an instance of PreparedData') + + if not isinstance(batch_size, int): + raise TypeError('batch_size must be an int') diff --git a/tests/icosagon/test_databatch.py b/tests/icosagon/test_databatch.py new file mode 100644 index 0000000..2a35843 --- /dev/null +++ b/tests/icosagon/test_databatch.py @@ -0,0 +1,81 @@ +from icosagon.databatch import DataBatcher, \ + BatchedData +from icosagon.data import Data +from icosagon.trainprep import prepare_training, \ + TrainValTest +import torch + + +def _some_data(): + data = Data() + data.add_node_type('Foo', 100) + data.add_node_type('Bar', 500) + fam = data.add_relation_family('Foo-Bar', 0, 1, True) + adj_mat = torch.rand(100, 500).round().to_sparse() + fam.add_relation_type('Foo-Bar', adj_mat) + return data + + +def test_data_batcher_01(): + data = _some_data() + prep_d = prepare_training(data, TrainValTest(.8, .1, .1)) + batcher = DataBatcher(prep_d, 512) + + +def test_data_batcher_02(): + data = _some_data() + prep_d = prepare_training(data, TrainValTest(.8, .1, .1)) + batcher = DataBatcher(prep_d, 512) + for batch_d in batcher: + pass + + +def test_data_batcher_03(): + data = _some_data() + prep_d = prepare_training(data, TrainValTest(.8, .1, .1)) + batcher = DataBatcher(prep_d, 512) + for batch_d in batcher: + edges_list = [] + for fam in batch_d.relation_families: + for rel in fam.relation_types: + for edge_type in ['edges_pos', 'edges_neg', + 'edges_back_pos', 'edges_back_neg']: + for part_type in ['train', 'val', 'test']: + edges = getattr(getattr(rel, edge_type), part_type) + edges_list.append(edges) + assert sum([ 1 for edges in edges_list if len(edges) > 0 ]) == 1 + + +def test_data_batcher_04(): + data = _some_data() + prep_d = prepare_training(data, TrainValTest(.8, .1, .1)) + batcher = DataBatcher(prep_d, 512) + edges_list = [] + for batch_d in batcher: + for fam in batch_d.relation_families: + for rel in fam.relation_types: + for edge_type in ['edges_pos', 'edges_neg', + 'edges_back_pos', 'edges_back_neg']: + for part_type in ['train', 'val', 'test']: + edges = getattr(getattr(rel, edge_type), part_type) + edges_list.append(edges) + assert sum([ len(edges) for edges in edges_list ]) == \ + torch.sum(data.relation_families[0].relation_types[0].adjacency_matrix._values()) * 2 + + +def test_data_batcher_05(): + data = _some_data() + prep_d = prepare_training(data, TrainValTest(.8, .1, .1)) + batcher = DataBatcher(prep_d, 512) + for batch_d in batcher: + edges_list = [] + for fam in batch_d.relation_families: + for rel in fam.relation_types: + for edge_type in ['edges_pos', 'edges_neg', + 'edges_back_pos', 'edges_back_neg']: + for part_type in ['train', 'val', 'test']: + edges = getattr(getattr(rel, edge_type), part_type) + edges_list.append(edges) + assert all([ len(edges) <= 512 for edges in edges_list ]) + assert not all([ len(edges) == 0 for edges in edges_list ]) + print(sum(map(len, edges_list))) -- 2.26.2 From 45a18a46aa60e451430c79325d7d861bd6fa96f2 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 21 Jul 2020 14:26:16 +0200 Subject: [PATCH 157/227] Add shuffle to DataBatcher. --- src/icosagon/databatch.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/icosagon/databatch.py b/src/icosagon/databatch.py index b7c4a8f..6f96baf 100644 --- a/src/icosagon/databatch.py +++ b/src/icosagon/databatch.py @@ -2,6 +2,8 @@ from icosagon.trainprep import PreparedData, \ PreparedRelationFamily, \ PreparedRelationType, \ _empty_edge_list_tvt +import torch +import random class BatchedData(PreparedData): @@ -31,11 +33,13 @@ def batched_data_skeleton(data: PreparedData) -> BatchedData: class DataBatcher(object): - def __init__(self, data: PreparedData, batch_size: int) -> None: + def __init__(self, data: PreparedData, batch_size: int, + shuffle: bool = True) -> None: self._check_params(data, batch_size) self.data = data self.batch_size = batch_size + self.shuffle = shuffle # def batched_data_iter(self, fam_idx: int, rel_idx: int, # part_type: str) -> BatchedData: @@ -71,17 +75,34 @@ class DataBatcher(object): # yield batched_data def __iter__(self) -> BatchedData: + gen = self.shuffle_iter() \ + if self.shuffle \ + else self.iter_base() + + for batched_data in gen: + yield batched_data + + def iter_base(self) -> BatchedData: for i, fam in enumerate(self.data.relation_families): for k, rel in enumerate(fam.relation_types): for edge_type in ['edges_pos', 'edges_neg', 'edges_back_pos', 'edges_back_neg']: for part_type in ['train', 'val', 'test']: edges = getattr(getattr(rel, edge_type), part_type) + if self.shuffle: + perm = torch.randperm(len(edges)) + edges = edges[perm] for m in range(0, len(edges), self.batch_size): batched_data = batched_data_skeleton(self.data) setattr(getattr(batched_data.relation_families[i].relation_types[k], edge_type), part_type, edges[m : m + self.batch_size]) yield batched_data + def shuffle_iter(self) -> BatchedData: + res = list(self.iter_base()) + random.shuffle(res) + for batched_data in res: + yield batched_data + @staticmethod def _check_params(data, batch_size): if not isinstance(data, PreparedData): -- 2.26.2 From 5bc276eb6b2f29298e7fa49dcc10aa43880a2a1b Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 21 Jul 2020 14:56:43 +0200 Subject: [PATCH 158/227] Add BatchedData support to DecodeLayer. --- src/icosagon/convlayer.py | 2 +- src/icosagon/databatch.py | 5 ++++ src/icosagon/declayer.py | 19 +++++++++++-- tests/icosagon/test_databatch.py | 47 +++++++++++++++++++++++++++++++- 4 files changed, 68 insertions(+), 5 deletions(-) diff --git a/src/icosagon/convlayer.py b/src/icosagon/convlayer.py index 9b8e5ae..3c5b603 100644 --- a/src/icosagon/convlayer.py +++ b/src/icosagon/convlayer.py @@ -122,5 +122,5 @@ class DecagonLayer(torch.nn.Module): next_layer_repr[node_type_row] = sum(next_layer_repr[node_type_row]) next_layer_repr[node_type_row] = self.layer_activation(next_layer_repr[node_type_row]) - print('DecagonLayer.forward() took', time.time() - t) + # print('DecagonLayer.forward() took', time.time() - t) return next_layer_repr diff --git a/src/icosagon/databatch.py b/src/icosagon/databatch.py index 6f96baf..3602d6d 100644 --- a/src/icosagon/databatch.py +++ b/src/icosagon/databatch.py @@ -11,6 +11,11 @@ class BatchedData(PreparedData): super().__init__(*args, **kwargs) +class BatchedDataPointer(object): + def __init__(self, batched_data): + self.batched_data = batched_data + + def batched_data_skeleton(data: PreparedData) -> BatchedData: if not isinstance(data, PreparedData): raise TypeError('data must be an instance of PreparedData') diff --git a/src/icosagon/declayer.py b/src/icosagon/declayer.py index 13f751d..25d9c5f 100644 --- a/src/icosagon/declayer.py +++ b/src/icosagon/declayer.py @@ -17,6 +17,7 @@ from typing import Type, \ from .decode import DEDICOMDecoder from dataclasses import dataclass import time +from .databatch import BatchedDataPointer @dataclass @@ -43,6 +44,7 @@ class DecodeLayer(torch.nn.Module): data: PreparedData, keep_prob: float = 1., activation: Callable[[torch.Tensor], torch.Tensor] = torch.sigmoid, + batched_data_pointer: BatchedDataPointer = None, **kwargs) -> None: super().__init__(**kwargs) @@ -59,11 +61,19 @@ class DecodeLayer(torch.nn.Module): if not isinstance(data, PreparedData): raise TypeError('data must be an instance of PreparedData') + if batched_data_pointer is not None and \ + not isinstance(batched_data_pointer, BatchedDataPointer): + raise TypeError('batched_data_pointer must be an instance of BatchedDataPointer') + + # if batched_data_pointer is not None and not batched_data_pointer.compatible_with(data): + # raise ValueError('batched_data_pointer must be compatible with data') + self.input_dim = input_dim[0] self.output_dim = 1 self.data = data self.keep_prob = keep_prob self.activation = activation + self.batched_data_pointer = batched_data_pointer self.decoders = None self.build() @@ -88,13 +98,16 @@ class DecodeLayer(torch.nn.Module): tvt.append(dec(inputs_row, inputs_column, k)) tvt = TrainValTest(*tvt) pred.append(tvt) - print('DecodeLayer._get_tvt() took:', time.time() - start_time) + # print('DecodeLayer._get_tvt() took:', time.time() - start_time) return pred def forward(self, last_layer_repr: List[torch.Tensor]) -> List[List[torch.Tensor]]: t = time.time() res = [] - for i, fam in enumerate(self.data.relation_families): + data = self.batched_data_pointer.batched_data \ + if self.batched_data_pointer is not None \ + else self.data + for i, fam in enumerate(data.relation_families): fam_pred = [] for k, r in enumerate(fam.relation_types): pred = [] @@ -107,5 +120,5 @@ class DecodeLayer(torch.nn.Module): fam_pred = RelationFamilyPredictions(fam_pred) res.append(fam_pred) res = Predictions(res) - print('DecodeLayer.forward() took', time.time() - t) + # print('DecodeLayer.forward() took', time.time() - t) return res diff --git a/tests/icosagon/test_databatch.py b/tests/icosagon/test_databatch.py index 2a35843..b36b5da 100644 --- a/tests/icosagon/test_databatch.py +++ b/tests/icosagon/test_databatch.py @@ -1,9 +1,14 @@ from icosagon.databatch import DataBatcher, \ - BatchedData + BatchedData, \ + BatchedDataPointer, \ + batched_data_skeleton from icosagon.data import Data from icosagon.trainprep import prepare_training, \ TrainValTest +from icosagon.declayer import DecodeLayer +from icosagon.input import OneHotInputLayer import torch +import time def _some_data(): @@ -16,6 +21,16 @@ def _some_data(): return data +def _some_data_big(): + data = Data() + data.add_node_type('Foo', 2000) + data.add_node_type('Bar', 2100) + fam = data.add_relation_family('Foo-Bar', 0, 1, True) + adj_mat = torch.rand(2000, 2100).round().to_sparse() + fam.add_relation_type('Foo-Bar', adj_mat) + return data + + def test_data_batcher_01(): data = _some_data() prep_d = prepare_training(data, TrainValTest(.8, .1, .1)) @@ -79,3 +94,33 @@ def test_data_batcher_05(): assert all([ len(edges) <= 512 for edges in edges_list ]) assert not all([ len(edges) == 0 for edges in edges_list ]) print(sum(map(len, edges_list))) + + +def test_batch_decode_01(): + data = _some_data() + prep_d = prepare_training(data, TrainValTest(.8, .1, .1)) + batcher = DataBatcher(prep_d, 512) + ptr = BatchedDataPointer(batched_data_skeleton(prep_d)) + in_repr = [ torch.rand(100, 32), + torch.rand(500, 32) ] + dec_layer = DecodeLayer([ 32, 32 ], prep_d, batched_data_pointer=ptr) + t = time.time() + for batched_data in batcher: + ptr.batched_data = batched_data + _ = dec_layer(in_repr) + print('Elapsed:', time.time() - t) + + +def test_batch_decode_02(): + data = _some_data_big() + prep_d = prepare_training(data, TrainValTest(.8, .1, .1)) + batcher = DataBatcher(prep_d, 512) + ptr = BatchedDataPointer(batched_data_skeleton(prep_d)) + in_repr = [ torch.rand(2000, 32), + torch.rand(2100, 32) ] + dec_layer = DecodeLayer([ 32, 32 ], prep_d, batched_data_pointer=ptr) + t = time.time() + for batched_data in batcher: + ptr.batched_data = batched_data + _ = dec_layer(in_repr) + print('Elapsed:', time.time() - t) -- 2.26.2 From 54c7f1ae8f8829d6bfdc6a441f497e736f8b383f Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 21 Jul 2020 15:01:33 +0200 Subject: [PATCH 159/227] Fix a typo. --- src/icosagon/bulkdec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/icosagon/bulkdec.py b/src/icosagon/bulkdec.py index ea6dba8..cbf615e 100644 --- a/src/icosagon/bulkdec.py +++ b/src/icosagon/bulkdec.py @@ -44,7 +44,7 @@ def convert_decoder(dec): local_variation = torch.eye(dec.input_dim, dec.input_dim) local_variation = [ local_variation ] * dec.num_relation_types else: - raise TypeError('Unknown decoder type in covert_decoder()') + raise TypeError('Unknown decoder type in convert_decoder()') if not isinstance(local_variation, torch.Tensor): local_variation = map(lambda a: a.view(1, *a.shape), local_variation) -- 2.26.2 From 36fe246ff18757bc02649af6798246a51e9c7050 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 22 Jul 2020 12:51:30 +0200 Subject: [PATCH 160/227] Start working on fastconv. --- src/icosagon/fastconv.py | 110 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 src/icosagon/fastconv.py diff --git a/src/icosagon/fastconv.py b/src/icosagon/fastconv.py new file mode 100644 index 0000000..31428b2 --- /dev/null +++ b/src/icosagon/fastconv.py @@ -0,0 +1,110 @@ +from typing import List, \ + Union, \ + Callable +from .data import Data +from .trainprep import PreparedData +import torch +from .weights import init_glorot + + +class FastConvLayer(torch.nn.Module): + adjacency_matrix: List[torch.Tensor] + adjacency_matrix_backward: List[torch.Tensor] + weight: List[torch.Tensor] + weight_backward: List[torch.Tensor] + + def __init__(self, + input_dim: List[int], + output_dim: List[int], + data: Union[Data, PreparedData], + keep_prob: float = 1., + rel_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x + layer_activation: Callable[[torch.Tensor], torch.Tensor] = torch.nn.functional.relu, + **kwargs): + + super().__init__(**kwargs) + + self._check_params(input_dim, output_dim, data, keep_prob, + rel_activation, layer_activation) + + self.input_dim = input_dim + self.output_dim = output_dim + self.data = data + self.keep_prob = keep_prob + self.rel_activation = rel_activation + self.layer_activation = layer_activation + + self.adjacency_matrix = None + self.adjacency_matrix_backward = None + self.weight = None + self.weight_backward = None + self.build() + + def build(self): + self.adjacency_matrix = [] + self.adjacency_matrix_backward = [] + self.weight = [] + self.weight_backward = [] + for fam in self.data.relation_families: + adj_mat = [ rel.adjacency_matrix \ + for rel in fam.relation_types \ + if rel.adjacency_matrix is not None ] + adj_mat_back = [ rel.adjacency_matrix_backward \ + for rel in fam.relation_types \ + if rel.adjacency_matrix_backward is not None ] + weight = [ init_glorot(self.input_dim[fam.node_type_column], + self.output_dim[fam.node_type_row]) \ + for _ in range(len(adj_mat)) ] + weight_back = [ init_glorot(self.input_dim[fam.node_type_column], + self.output_dim[fam.node_type_row]) \ + for _ in range(len(adj_mat_back)) ] + adj_mat = torch.cat(adj_mat) \ + if len(adj_mat) > 0 \ + else None + adj_mat_back = torch.cat(adj_mat_back) \ + if len(adj_mat_back) > 0 \ + else None + self.adjacency_matrix.append(adj_mat) + self.adjacency_matrix_backward.append(adj_mat_back) + self.weight.append(weight) + self.weight_back.append(weight_back) + + def forward(self, prev_layer_repr): + for i, fam in enumerate(self.data.relation_families): + repr_row = prev_layer_repr[fam.node_type_row] + repr_column = prev_layer_repr[fam.node_type_column] + + adj_mat = self.adjacency_matrix[i] + adj_mat_back = self.adjacency_matrix_backward[i] + + if adj_mat is not None: + x = dropout(repr_column, keep_prob=self.keep_prob) + x = torch.sparse.mm(x, self.weight[i]) \ + if x.is_sparse \ + else torch.mm(x, self.weight[i]) + x = torch.sparse.mm(adj_mat, repr_row) \ + if adj_mat.is_sparse \ + else torch.mm(adj_mat, repr_row) + x = self.rel_activation(x) + x = x.view(len(fam.relation_types), len(repr_row), -1) + + if adj_mat_back is not None: + x = torch.sparse.mm(adj_mat_back, repr_row) \ + if adj_mat_back.is_sparse \ + else torch.mm(adj_mat_back, repr_row) + + @staticmethod + def _check_params(input_dim, output_dim, data, keep_prob, + rel_activation, layer_activation): + + if not isinstance(input_dim, list): + raise ValueError('input_dim must be a list') + + if not output_dim: + raise ValueError('output_dim must be specified') + + if not isinstance(output_dim, list): + output_dim = [output_dim] * len(data.node_types) + + if not isinstance(data, Data) and not isinstance(data, PreparedData): + raise ValueError('data must be of type Data or PreparedData') -- 2.26.2 From dd41ba9bd004e8f1fa86a02228172a7218e9760f Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 22 Jul 2020 14:28:17 +0200 Subject: [PATCH 161/227] Add matrix-multiply illustration. --- docs/matrix-multiply.svg | 839 +++++++++++++++++++++++++++++++++++++++ src/icosagon/fastconv.py | 61 +++ 2 files changed, 900 insertions(+) create mode 100644 docs/matrix-multiply.svg diff --git a/docs/matrix-multiply.svg b/docs/matrix-multiply.svg new file mode 100644 index 0000000..4259ec3 --- /dev/null +++ b/docs/matrix-multiply.svg @@ -0,0 +1,839 @@ + + + + + + + + + + image/svg+xml + + + + + + + + x + + w1 + + w2 + + w3 + + w4 + + * + = + + x*w1 + + x*w2 + + x*w3 + + x*w4 + + + x*w1 + + x*w2 + + x*w3 + + x*w4 + + * + + + + + + A1 + A2 + A3 + A4 + = + + + + + + + + + + + + + + + + + + A1*x*w1 + A1*x*w2 + A1*x*w3 + A1*x*w4 + A2*x*w1 + A2*x*w2 + A2*x*w3 + A2*x*w4 + A3*x*w1 + A3*x*w2 + A3*x*w3 + A3*x*w4 + A4*x*w1 + A4*x*w2 + A4*x*w3 + A4*x*w4 + + + diff --git a/src/icosagon/fastconv.py b/src/icosagon/fastconv.py index 31428b2..ba27660 100644 --- a/src/icosagon/fastconv.py +++ b/src/icosagon/fastconv.py @@ -7,6 +7,67 @@ import torch from .weights import init_glorot +def _cat(matrices: List[torch.Tensor]): + if len(matrices) == 0: + raise ValueError('Empty list passed to _cat()') + + n = sum(a.is_sparse for a in matrices) + if n != 0 and n != len(matrices): + raise ValueError('All matrices must have the same layout (dense or sparse)') + + if not all(a.shape[1:] == matrices[0].shape[1:]): + raise ValueError('All matrices must have the same dimensions apart from dimension 0') + + if not matrices[0].is_sparse: + return torch.cat(matrices) + + total_rows = sum(a.shape[0] for a in matrices) + indices = [] + values = [] + row_offset = 0 + + for a in matrices: + ind = a._indices().clone() + val = a._values() + ind[0] += row_offset + ind = ind.transpose(0, 1) + indices.append(ind) + values.append(val) + row_offset += a.shape[0] + + indices = torch.cat(indices).transpose(0, 1) + values = torch.cat(values) + + res = _sparse_coo_tensor(indices, values) + return res + + +class FastGraphConv(torch.nn.Module): + def __init__(self, + in_channels: int, + out_channels: int, + adjacency_matrix: List[torch.Tensor], + **kwargs): + + self.in_channels = in_channels + self.out_channels = out_channels + self.weight = torch.cat([ + init_glorot(in_channels, out_channels) \ + for _ in adjacency_matrix + ], dim=1) + self.adjacency_matrix = _cat(adjacency_matrix) + + def forward(self, x): + x = torch.sparse.mm(x, self.weight) \ + if x.is_sparse \ + else torch.mm(x, self.weight) + x = torch.sparse.mm(self.adjacency_matrix, x) \ + if self.adjacency_matrix.is_sparse \ + else torch.mm(self.adjacency_matrix, x) + return x + + + class FastConvLayer(torch.nn.Module): adjacency_matrix: List[torch.Tensor] adjacency_matrix_backward: List[torch.Tensor] -- 2.26.2 From 5770f7ce99154483b088e5128c79ebd400345c88 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 23 Jul 2020 20:46:31 +0200 Subject: [PATCH 162/227] Update matrix-multiply with an idea that will actually work. --- docs/matrix-multiply.svg | 557 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 552 insertions(+), 5 deletions(-) diff --git a/docs/matrix-multiply.svg b/docs/matrix-multiply.svg index 4259ec3..5853f0f 100644 --- a/docs/matrix-multiply.svg +++ b/docs/matrix-multiply.svg @@ -26,13 +26,13 @@ inkscape:pageopacity="0.0" inkscape:pageshadow="2" inkscape:zoom="1.06" - inkscape:cx="442.13641" - inkscape:cy="826.66981" + inkscape:cx="366.58965" + inkscape:cy="397.63915" inkscape:document-units="mm" inkscape:current-layer="layer1" showgrid="false" - inkscape:window-width="1920" - inkscape:window-height="1001" + inkscape:window-width="2560" + inkscape:window-height="1361" inkscape:window-x="-9" inkscape:window-y="-9" inkscape:window-maximized="1" @@ -403,7 +403,7 @@ height="19.589043" width="67.631081" id="rect968" - style="opacity:1;vector-effect:none;fill:none;fill-opacity:1;stroke:#00c3de;stroke-width:0.72899967;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke" + style="opacity:1;vector-effect:none;fill:none;fill-opacity:1;stroke:#00c3de;stroke-width:0.72899997;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:2.91599989,2.91599989;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke;font-variant-east_asian:normal" transform="rotate(90)" /> + + x*w1 + + x*w2 + + x*w3 + + x*w4 + + * + + + + + + A4 offset for x*w4 + A3 offset for x*w3 + A2 offset for x*w2 + A1 offset for x*w1 + + A1*(x*w1) + + + + + = + A2*(x*w2) + A3*(x*w3) + A4*(x*w4) + + x + + w1 + + w2 + + w3 + + w4 + + * + = + + x*w1 + + x*w2 + + x*w3 + + x*w4 + + + x + or -- 2.26.2 From 3ecc5a79d744e3580586e694bbc383f1f63bdfef Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 23 Jul 2020 20:57:03 +0200 Subject: [PATCH 163/227] Update matrix-multiply. --- docs/matrix-multiply.svg | 168 ++++++++++++++++++++++++++++----------- 1 file changed, 122 insertions(+), 46 deletions(-) diff --git a/docs/matrix-multiply.svg b/docs/matrix-multiply.svg index 5853f0f..a2af2a5 100644 --- a/docs/matrix-multiply.svg +++ b/docs/matrix-multiply.svg @@ -25,9 +25,9 @@ borderopacity="1.0" inkscape:pageopacity="0.0" inkscape:pageshadow="2" - inkscape:zoom="1.06" - inkscape:cx="366.58965" - inkscape:cy="397.63915" + inkscape:zoom="1.4990664" + inkscape:cx="189.50273" + inkscape:cy="386.13684" inkscape:document-units="mm" inkscape:current-layer="layer1" showgrid="false" @@ -53,6 +53,38 @@ inkscape:label="Layer 1" inkscape:groupmode="layer" id="layer1"> + + + + x*w1 x*w x*w3 x*w4 * @@ -946,33 +978,33 @@ style="opacity:1;vector-effect:none;fill:none;fill-opacity:1;stroke:#00c3de;stroke-width:0.72899967;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke" id="rect1233" width="12.935501" - height="61.847054" + height="54.434132" x="233.25441" - y="-82.344681" /> + y="-74.931732" /> + y="-75.035126" /> + y="-78.849915" /> 1 A = AAAor + sparse + dense + dense + sparse -- 2.26.2 From 058d4d43fb83c6b8ada8d1b73f5077e5fb5d2679 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 24 Jul 2020 11:02:09 +0200 Subject: [PATCH 164/227] Add _sparse_diag_cat(). --- src/icosagon/fastconv.py | 33 ++++++++++++++++++++++++++++++++- tests/icosagon/test_fastconv.py | 17 +++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 tests/icosagon/test_fastconv.py diff --git a/src/icosagon/fastconv.py b/src/icosagon/fastconv.py index ba27660..0dd5074 100644 --- a/src/icosagon/fastconv.py +++ b/src/icosagon/fastconv.py @@ -5,6 +5,37 @@ from .data import Data from .trainprep import PreparedData import torch from .weights import init_glorot +from .normalize import _sparse_coo_tensor + + +def _sparse_diag_cat(matrices: List[torch.Tensor]): + if len(matrices) == 0: + raise ValueError('The list of matrices must be non-empty') + + if not all(m.is_sparse for m in matrices): + raise ValueError('All matrices must be sparse') + + if not all(len(m.shape) == 2 for m in matrices): + raise ValueError('All matrices must be 2D') + + indices = [] + values = [] + row_offset = 0 + col_offset = 0 + + for m in matrices: + ind = m._indices().clone() + ind[0] += row_offset + ind[1] += col_offset + indices.append(ind) + values.append(m._values()) + row_offset += m.shape[0] + col_offset += m.shape[1] + + indices = torch.cat(indices, dim=1) + values = torch.cat(values) + + return _sparse_coo_tensor(indices, values, size=(row_offset, col_offset)) def _cat(matrices: List[torch.Tensor]): @@ -79,7 +110,7 @@ class FastConvLayer(torch.nn.Module): output_dim: List[int], data: Union[Data, PreparedData], keep_prob: float = 1., - rel_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x + rel_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, layer_activation: Callable[[torch.Tensor], torch.Tensor] = torch.nn.functional.relu, **kwargs): diff --git a/tests/icosagon/test_fastconv.py b/tests/icosagon/test_fastconv.py new file mode 100644 index 0000000..963df9b --- /dev/null +++ b/tests/icosagon/test_fastconv.py @@ -0,0 +1,17 @@ +from icosagon.fastconv import _sparse_diag_cat +import torch + + +def test_sparse_diag_cat_01(): + matrices = [ torch.rand(5, 10).round() for _ in range(7) ] + ground_truth = torch.zeros(35, 70) + ground_truth[0:5, 0:10] = matrices[0] + ground_truth[5:10, 10:20] = matrices[1] + ground_truth[10:15, 20:30] = matrices[2] + ground_truth[15:20, 30:40] = matrices[3] + ground_truth[20:25, 40:50] = matrices[4] + ground_truth[25:30, 50:60] = matrices[5] + ground_truth[30:35, 60:70] = matrices[6] + res = _sparse_diag_cat([ m.to_sparse() for m in matrices ]) + res = res.to_dense() + assert torch.all(res == ground_truth) -- 2.26.2 From 5e2818fb8d2c21d5262fbc9d0d510a9696e45976 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 24 Jul 2020 11:27:16 +0200 Subject: [PATCH 165/227] Add test_sparse_diag_cat_02(). --- tests/icosagon/test_fastconv.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/icosagon/test_fastconv.py b/tests/icosagon/test_fastconv.py index 963df9b..8aaf00d 100644 --- a/tests/icosagon/test_fastconv.py +++ b/tests/icosagon/test_fastconv.py @@ -15,3 +15,22 @@ def test_sparse_diag_cat_01(): res = _sparse_diag_cat([ m.to_sparse() for m in matrices ]) res = res.to_dense() assert torch.all(res == ground_truth) + + +def test_sparse_diag_cat_02(): + x = [ torch.rand(5, 10).round() for _ in range(7) ] + a = [ m.to_sparse() for m in x ] + a = _sparse_diag_cat(a) + b = torch.rand(70, 64) + res = torch.sparse.mm(a, b) + + ground_truth = torch.zeros(35, 64) + ground_truth[0:5, :] = torch.mm(x[0], b[0:10]) + ground_truth[5:10, :] = torch.mm(x[1], b[10:20]) + ground_truth[10:15, :] = torch.mm(x[2], b[20:30]) + ground_truth[15:20, :] = torch.mm(x[3], b[30:40]) + ground_truth[20:25, :] = torch.mm(x[4], b[40:50]) + ground_truth[25:30, :] = torch.mm(x[5], b[50:60]) + ground_truth[30:35, :] = torch.mm(x[6], b[60:70]) + + assert torch.all(res == ground_truth) -- 2.26.2 From 271fba00041f6d7d8ef82605a65afbf14a21e1ce Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 24 Jul 2020 12:27:21 +0200 Subject: [PATCH 166/227] Add test_cat_01() and test_cat_02(). --- src/icosagon/fastconv.py | 4 ++-- tests/icosagon/test_fastconv.py | 26 +++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/icosagon/fastconv.py b/src/icosagon/fastconv.py index 0dd5074..74e0319 100644 --- a/src/icosagon/fastconv.py +++ b/src/icosagon/fastconv.py @@ -46,7 +46,7 @@ def _cat(matrices: List[torch.Tensor]): if n != 0 and n != len(matrices): raise ValueError('All matrices must have the same layout (dense or sparse)') - if not all(a.shape[1:] == matrices[0].shape[1:]): + if not all(a.shape[1:] == matrices[0].shape[1:] for a in matrices): raise ValueError('All matrices must have the same dimensions apart from dimension 0') if not matrices[0].is_sparse: @@ -69,7 +69,7 @@ def _cat(matrices: List[torch.Tensor]): indices = torch.cat(indices).transpose(0, 1) values = torch.cat(values) - res = _sparse_coo_tensor(indices, values) + res = _sparse_coo_tensor(indices, values, size=(row_offset, matrices[0].shape[1])) return res diff --git a/tests/icosagon/test_fastconv.py b/tests/icosagon/test_fastconv.py index 8aaf00d..799da5d 100644 --- a/tests/icosagon/test_fastconv.py +++ b/tests/icosagon/test_fastconv.py @@ -1,4 +1,6 @@ -from icosagon.fastconv import _sparse_diag_cat +from icosagon.fastconv import _sparse_diag_cat, \ + _cat +from icosagon.data import _equal import torch @@ -34,3 +36,25 @@ def test_sparse_diag_cat_02(): ground_truth[30:35, :] = torch.mm(x[6], b[60:70]) assert torch.all(res == ground_truth) + + +def test_cat_01(): + matrices = [ torch.rand(5, 10) for _ in range(7) ] + res = _cat(matrices) + assert res.shape == (35, 10) + assert not res.is_sparse + ground_truth = torch.zeros(35, 10) + for i in range(7): + ground_truth[i*5:(i+1)*5, :] = matrices[i] + assert torch.all(res == ground_truth) + + +def test_cat_02(): + matrices = [ torch.rand(5, 10) for _ in range(7) ] + ground_truth = torch.zeros(35, 10) + for i in range(7): + ground_truth[i*5:(i+1)*5, :] = matrices[i] + res = _cat([ m.to_sparse() for m in matrices ]) + assert res.shape == (35, 10) + assert res.is_sparse + assert torch.all(res.to_dense() == ground_truth) -- 2.26.2 From e2ae97add31b6d38f1f982d8701c8f52726dba2b Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 24 Jul 2020 13:26:30 +0200 Subject: [PATCH 167/227] Work on FastGraphConv. --- src/icosagon/fastconv.py | 87 ++++++++++++++++++++++++++++++++-------- 1 file changed, 71 insertions(+), 16 deletions(-) diff --git a/src/icosagon/fastconv.py b/src/icosagon/fastconv.py index 74e0319..02086c4 100644 --- a/src/icosagon/fastconv.py +++ b/src/icosagon/fastconv.py @@ -1,11 +1,14 @@ from typing import List, \ Union, \ Callable -from .data import Data -from .trainprep import PreparedData +from .data import Data, \ + RelationFamily +from .trainprep import PreparedData, \ + PreparedRelationFamily import torch from .weights import init_glorot from .normalize import _sparse_coo_tensor +import types def _sparse_diag_cat(matrices: List[torch.Tensor]): @@ -75,28 +78,80 @@ def _cat(matrices: List[torch.Tensor]): class FastGraphConv(torch.nn.Module): def __init__(self, - in_channels: int, - out_channels: int, - adjacency_matrix: List[torch.Tensor], - **kwargs): + in_channels: List[int], + out_channels: List[int], + data: Union[Data, PreparedData], + relation_family: Union[RelationFamily, PreparedRelationFamily] + keep_prob: float = 1., + acivation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, + **kwargs) -> None: + + in_channels = int(in_channels) + out_channels = int(out_channels) + if not isinstance(data, Data) and not isinstance(data, PreparedData): + raise TypeError('data must be an instance of Data or PreparedData') + if not isinstance(relation_family, RelationFamily) and \ + not isinstance(relation_family, PreparedRelationFamily): + raise TypeError('relation_family must be an instance of RelationFamily or PreparedRelationFamily') + keep_prob = float(keep_prob) + if not isinstance(activation, types.FunctionType): + raise TypeError('activation must be a function') + + n_nodes_row = data.node_types[relation_family.node_type_row].count + n_nodes_column = data.node_types[relation_family.node_type_column].count self.in_channels = in_channels self.out_channels = out_channels + self.data = data + self.relation_family = relation_family + self.keep_prob = keep_prob + self.activation = activation + self.weight = torch.cat([ init_glorot(in_channels, out_channels) \ - for _ in adjacency_matrix + for _ in range(len(relation_family.relation_types)) ], dim=1) - self.adjacency_matrix = _cat(adjacency_matrix) - def forward(self, x): - x = torch.sparse.mm(x, self.weight) \ - if x.is_sparse \ - else torch.mm(x, self.weight) - x = torch.sparse.mm(self.adjacency_matrix, x) \ - if self.adjacency_matrix.is_sparse \ - else torch.mm(self.adjacency_matrix, x) - return x + self.weight_backward = torch.cat([ + init_glorot(in_channels, out_channels) \ + for _ in range(len(relation_family.relation_types)) + ], dim=1) + self.adjacency_matrix = _sparse_diag_cat([ + rel.adjacency_matrix \ + if rel.adjacency_matrix is not None \ + else _sparse_coo_tensor([], [], size=(n_nodes_row, n_nodes_column)) \ + for rel in relation_family.relation_types ]) + + self.adjacency_matrix_backward = _sparse_diag_cat([ + rel.adjacency_matrix_backward \ + if rel.adjacency_matrix_backward is not None \ + else _sparse_coo_tensor([], [], size=(n_nodes_column, n_nodes_row)) \ + for rel in relation_family.relation_types ]) + + def forward(self, prev_layer_repr: List[torch.Tensor]) -> List[torch.Tensor]: + repr_row = prev_layer_repr[self.relation_family.node_type_row] + repr_column = prev_layer_repr[self.relation_family.node_type_column] + + new_repr_row = torch.sparse.mm(repr_column, self.weight) \ + if repr_column.is_sparse \ + else torch.mm(repr_column, self.weight) + new_repr_row = torch.sparse.mm(self.adjacency_matrix, new_repr_row) \ + if self.adjacency_matrix.is_sparse \ + else torch.mm(self.adjacency_matrix, new_repr_row) + new_repr_row = new_repr_row.view(len(self.relation_family.relation_types), + len(repr_row), self.out_channels) + + new_repr_column = torch.sparse.mm(repr_row, self.weight) \ + if repr_row.is_sparse \ + else torch.mm(repr_row, self.weight) + new_repr_column = torch.sparse.mm(self.adjacency_matrix_backward, new_repr_column) \ + if self.adjacency_matrix_backward.is_sparse \ + else torch.mm(self.adjacency_matrix_backward, new_repr_column) + new_repr_column = new_repr_column.view(len(self.relation_family.relation_types), + len(repr_column), self.out_channels) + + return (new_repr_row, new_repr_column) class FastConvLayer(torch.nn.Module): -- 2.26.2 From fcaccc8730086135246b2236033145cc21eaed46 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 24 Jul 2020 13:28:11 +0200 Subject: [PATCH 168/227] Fix a typo. --- src/icosagon/fastconv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/icosagon/fastconv.py b/src/icosagon/fastconv.py index 02086c4..6f90eb8 100644 --- a/src/icosagon/fastconv.py +++ b/src/icosagon/fastconv.py @@ -81,7 +81,7 @@ class FastGraphConv(torch.nn.Module): in_channels: List[int], out_channels: List[int], data: Union[Data, PreparedData], - relation_family: Union[RelationFamily, PreparedRelationFamily] + relation_family: Union[RelationFamily, PreparedRelationFamily], keep_prob: float = 1., acivation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, **kwargs) -> None: -- 2.26.2 From 1a303f1a5104e0c47e198e957931b7949614b602 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 24 Jul 2020 14:20:29 +0200 Subject: [PATCH 169/227] Add test_fast_graph_conv_01() and test_fast_graph_conv_02(). --- src/icosagon/fastconv.py | 93 +++++++++++++-------------------- tests/icosagon/test_fastconv.py | 30 ++++++++++- 2 files changed, 66 insertions(+), 57 deletions(-) diff --git a/src/icosagon/fastconv.py b/src/icosagon/fastconv.py index 6f90eb8..81519fb 100644 --- a/src/icosagon/fastconv.py +++ b/src/icosagon/fastconv.py @@ -78,80 +78,61 @@ def _cat(matrices: List[torch.Tensor]): class FastGraphConv(torch.nn.Module): def __init__(self, - in_channels: List[int], - out_channels: List[int], - data: Union[Data, PreparedData], - relation_family: Union[RelationFamily, PreparedRelationFamily], + in_channels: int, + out_channels: int, + adjacency_matrices: List[torch.Tensor], keep_prob: float = 1., - acivation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, + activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, **kwargs) -> None: + super().__init__(**kwargs) + in_channels = int(in_channels) out_channels = int(out_channels) - if not isinstance(data, Data) and not isinstance(data, PreparedData): - raise TypeError('data must be an instance of Data or PreparedData') - if not isinstance(relation_family, RelationFamily) and \ - not isinstance(relation_family, PreparedRelationFamily): - raise TypeError('relation_family must be an instance of RelationFamily or PreparedRelationFamily') + if not isinstance(adjacency_matrices, list): + raise TypeError('adjacency_matrices must be a list') + if len(adjacency_matrices) == 0: + raise ValueError('adjacency_matrices must not be empty') + if not all(isinstance(m, torch.Tensor) for m in adjacency_matrices): + raise TypeError('adjacency_matrices elements must be of class torch.Tensor') + if not all(m.is_sparse for m in adjacency_matrices): + raise ValueError('adjacency_matrices elements must be sparse') keep_prob = float(keep_prob) if not isinstance(activation, types.FunctionType): raise TypeError('activation must be a function') - n_nodes_row = data.node_types[relation_family.node_type_row].count - n_nodes_column = data.node_types[relation_family.node_type_column].count - self.in_channels = in_channels self.out_channels = out_channels - self.data = data - self.relation_family = relation_family + self.adjacency_matrices = adjacency_matrices self.keep_prob = keep_prob self.activation = activation - self.weight = torch.cat([ - init_glorot(in_channels, out_channels) \ - for _ in range(len(relation_family.relation_types)) - ], dim=1) + self.num_row_nodes = len(adjacency_matrices[0]) + self.num_relation_types = len(adjacency_matrices) + + self.adjacency_matrices = _sparse_diag_cat(adjacency_matrices) - self.weight_backward = torch.cat([ + self.weights = torch.cat([ init_glorot(in_channels, out_channels) \ - for _ in range(len(relation_family.relation_types)) + for _ in range(self.num_relation_types) ], dim=1) - self.adjacency_matrix = _sparse_diag_cat([ - rel.adjacency_matrix \ - if rel.adjacency_matrix is not None \ - else _sparse_coo_tensor([], [], size=(n_nodes_row, n_nodes_column)) \ - for rel in relation_family.relation_types ]) - - self.adjacency_matrix_backward = _sparse_diag_cat([ - rel.adjacency_matrix_backward \ - if rel.adjacency_matrix_backward is not None \ - else _sparse_coo_tensor([], [], size=(n_nodes_column, n_nodes_row)) \ - for rel in relation_family.relation_types ]) - - def forward(self, prev_layer_repr: List[torch.Tensor]) -> List[torch.Tensor]: - repr_row = prev_layer_repr[self.relation_family.node_type_row] - repr_column = prev_layer_repr[self.relation_family.node_type_column] - - new_repr_row = torch.sparse.mm(repr_column, self.weight) \ - if repr_column.is_sparse \ - else torch.mm(repr_column, self.weight) - new_repr_row = torch.sparse.mm(self.adjacency_matrix, new_repr_row) \ - if self.adjacency_matrix.is_sparse \ - else torch.mm(self.adjacency_matrix, new_repr_row) - new_repr_row = new_repr_row.view(len(self.relation_family.relation_types), - len(repr_row), self.out_channels) - - new_repr_column = torch.sparse.mm(repr_row, self.weight) \ - if repr_row.is_sparse \ - else torch.mm(repr_row, self.weight) - new_repr_column = torch.sparse.mm(self.adjacency_matrix_backward, new_repr_column) \ - if self.adjacency_matrix_backward.is_sparse \ - else torch.mm(self.adjacency_matrix_backward, new_repr_column) - new_repr_column = new_repr_column.view(len(self.relation_family.relation_types), - len(repr_column), self.out_channels) - - return (new_repr_row, new_repr_column) + def forward(self, x) -> torch.Tensor: + if self.keep_prob < 1.: + x = dropout(x, self.keep_prob) + res = torch.sparse.mm(x, self.weights) \ + if x.is_sparse \ + else torch.mm(x, self.weights) + res = torch.split(res, res.shape[1] // self.num_relation_types, dim=1) + res = torch.cat(res) + res = torch.sparse.mm(self.adjacency_matrices, res) \ + if self.adjacency_matrices.is_sparse \ + else torch.mm(self.adjacency_matrices, res) + res = res.view(self.num_relation_types, self.num_row_nodes, self.out_channels) + if self.activation is not None: + res = self.activation(res) + + return res class FastConvLayer(torch.nn.Module): diff --git a/tests/icosagon/test_fastconv.py b/tests/icosagon/test_fastconv.py index 799da5d..407248a 100644 --- a/tests/icosagon/test_fastconv.py +++ b/tests/icosagon/test_fastconv.py @@ -1,7 +1,10 @@ from icosagon.fastconv import _sparse_diag_cat, \ - _cat + _cat, \ + FastGraphConv from icosagon.data import _equal import torch +import pdb +import time def test_sparse_diag_cat_01(): @@ -58,3 +61,28 @@ def test_cat_02(): assert res.shape == (35, 10) assert res.is_sparse assert torch.all(res.to_dense() == ground_truth) + + +def test_fast_graph_conv_01(): + # pdb.set_trace() + adj_mats = [ torch.rand(10, 15).round().to_sparse() \ + for _ in range(23) ] + fgc = FastGraphConv(32, 64, adj_mats) + in_repr = torch.rand(15, 32) + _ = fgc(in_repr) + + +def test_fast_graph_conv_02(): + t = time.time() + m = (torch.rand(2000, 2000) < .001).to(torch.float32).to_sparse() + adj_mats = [ m for _ in range(1300) ] + print('Generating adj_mats took:', time.time() - t) + t = time.time() + fgc = FastGraphConv(32, 64, adj_mats) + print('FGC constructor took:', time.time() - t) + in_repr = torch.rand(2000, 32) + + for _ in range(3): + t = time.time() + _ = fgc(in_repr) + print('FGC forward pass took:', time.time() - t) -- 2.26.2 From c6f4b8779decbf6573e2d8d3726efd59324794f8 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 27 Jul 2020 17:01:53 +0200 Subject: [PATCH 170/227] Add test_fast_graph_conv_03() and test_fast_graph_conv_04(). --- src/icosagon/fastconv.py | 16 ++++-- tests/icosagon/test_fastconv.py | 87 ++++++++++++++++++++++++++++++++- 2 files changed, 97 insertions(+), 6 deletions(-) diff --git a/src/icosagon/fastconv.py b/src/icosagon/fastconv.py index 81519fb..8838c9a 100644 --- a/src/icosagon/fastconv.py +++ b/src/icosagon/fastconv.py @@ -195,7 +195,7 @@ class FastConvLayer(torch.nn.Module): self.adjacency_matrix.append(adj_mat) self.adjacency_matrix_backward.append(adj_mat_back) self.weight.append(weight) - self.weight_back.append(weight_back) + self.weight_backward.append(weight_back) def forward(self, prev_layer_repr): for i, fam in enumerate(self.data.relation_families): @@ -210,16 +210,22 @@ class FastConvLayer(torch.nn.Module): x = torch.sparse.mm(x, self.weight[i]) \ if x.is_sparse \ else torch.mm(x, self.weight[i]) - x = torch.sparse.mm(adj_mat, repr_row) \ + x = torch.sparse.mm(adj_mat, x) \ if adj_mat.is_sparse \ - else torch.mm(adj_mat, repr_row) + else torch.mm(adj_mat, x) x = self.rel_activation(x) x = x.view(len(fam.relation_types), len(repr_row), -1) if adj_mat_back is not None: - x = torch.sparse.mm(adj_mat_back, repr_row) \ + x = dropout(repr_column, keep_prob=self.keep_prob) + x = torch.sparse.mm(x, self.weight_backward[i]) \ + if x.is_sparse \ + else torch.mm(x, self.weight_backward[i]) + x = torch.sparse.mm(adj_mat_back, x) \ if adj_mat_back.is_sparse \ - else torch.mm(adj_mat_back, repr_row) + else torch.mm(adj_mat_back, x) + x = self.rel_activation(x) + x = x.view(len(fam.relation_types), len(repr_row), -1) @staticmethod def _check_params(input_dim, output_dim, data, keep_prob, diff --git a/tests/icosagon/test_fastconv.py b/tests/icosagon/test_fastconv.py index 407248a..742173d 100644 --- a/tests/icosagon/test_fastconv.py +++ b/tests/icosagon/test_fastconv.py @@ -1,10 +1,47 @@ from icosagon.fastconv import _sparse_diag_cat, \ _cat, \ - FastGraphConv + FastGraphConv, \ + FastConvLayer from icosagon.data import _equal import torch import pdb import time +from icosagon.data import Data +from icosagon.input import OneHotInputLayer +from icosagon.convlayer import DecagonLayer + + +def _make_symmetric(x: torch.Tensor): + x = (x + x.transpose(0, 1)) / 2 + return x + + +def _symmetric_random(n_rows, n_columns): + return _make_symmetric(torch.rand((n_rows, n_columns), + dtype=torch.float32).round()) + + +def _some_data_with_interactions(): + d = Data() + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + + fam = d.add_relation_family('Drug-Gene', 1, 0, True) + fam.add_relation_type('Target', + torch.rand((100, 1000), dtype=torch.float32).round()) + + fam = d.add_relation_family('Gene-Gene', 0, 0, True) + fam.add_relation_type('Interaction', + _symmetric_random(1000, 1000)) + + fam = d.add_relation_family('Drug-Drug', 1, 1, True) + fam.add_relation_type('Side Effect: Nausea', + _symmetric_random(100, 100)) + fam.add_relation_type('Side Effect: Infertility', + _symmetric_random(100, 100)) + fam.add_relation_type('Side Effect: Death', + _symmetric_random(100, 100)) + return d def test_sparse_diag_cat_01(): @@ -86,3 +123,51 @@ def test_fast_graph_conv_02(): t = time.time() _ = fgc(in_repr) print('FGC forward pass took:', time.time() - t) + + +def test_fast_graph_conv_03(): + adj_mat = [ + [ 0, 0, 1, 0, 1 ], + [ 0, 1, 0, 1, 0 ], + [ 1, 0, 1, 0, 0 ] + ] + in_repr = torch.rand(5, 32) + adj_mat = torch.tensor(adj_mat, dtype=torch.float32) + fgc = FastGraphConv(32, 64, [ adj_mat.to_sparse() ]) + out_repr = fgc(in_repr) + assert out_repr.shape == (1, 3, 64) + assert (torch.mm(adj_mat, torch.mm(in_repr, fgc.weights)).view(1, 3, 64) == out_repr).all() + + +def test_fast_graph_conv_04(): + adj_mat = [ + [ 0, 0, 1, 0, 1 ], + [ 0, 1, 0, 1, 0 ], + [ 1, 0, 1, 0, 0 ] + ] + in_repr = torch.rand(5, 32) + adj_mat = torch.tensor(adj_mat, dtype=torch.float32) + fgc = FastGraphConv(32, 64, [ adj_mat.to_sparse(), adj_mat.to_sparse() ]) + out_repr = fgc(in_repr) + assert out_repr.shape == (2, 3, 64) + adj_mat_1 = torch.zeros(adj_mat.shape[0] * 2, adj_mat.shape[1] * 2) + adj_mat_1[0:3, 0:5] = adj_mat + adj_mat_1[3:6, 5:10] = adj_mat + res = torch.mm(in_repr, fgc.weights) + res = torch.split(res, res.shape[1] // 2, dim=1) + res = torch.cat(res) + res = torch.mm(adj_mat_1, res) + assert (res.view(2, 3, 64) == out_repr).all() + + +def test_fast_conv_layer_01(): + d = _some_data_with_interactions() + in_layer = OneHotInputLayer(d) + + d_layer = DecagonLayer(in_layer.output_dim, [32, 32], d) + seq_1 = torch.nn.Sequential(in_layer, d_layer) + out_repr_1 = seq_1(None) + + conv_layer = FastConvLayer(in_layer.output_dim, [32, 32], d) + seq_2 = torch.nn.Sequential(in_layer, conv_layer) + out_repr_2 = seq_2(None) -- 2.26.2 From aa3cc1f3ad62cd0480932d2beebac14892bb1821 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 28 Jul 2020 10:20:09 +0200 Subject: [PATCH 171/227] Reimplement FastConvLayer. --- src/icosagon/fastconv.py | 137 +++++++++++++++++++++------------------ 1 file changed, 74 insertions(+), 63 deletions(-) diff --git a/src/icosagon/fastconv.py b/src/icosagon/fastconv.py index 8838c9a..038e2fc 100644 --- a/src/icosagon/fastconv.py +++ b/src/icosagon/fastconv.py @@ -136,11 +136,6 @@ class FastGraphConv(torch.nn.Module): class FastConvLayer(torch.nn.Module): - adjacency_matrix: List[torch.Tensor] - adjacency_matrix_backward: List[torch.Tensor] - weight: List[torch.Tensor] - weight_backward: List[torch.Tensor] - def __init__(self, input_dim: List[int], output_dim: List[int], @@ -162,70 +157,86 @@ class FastConvLayer(torch.nn.Module): self.rel_activation = rel_activation self.layer_activation = layer_activation - self.adjacency_matrix = None - self.adjacency_matrix_backward = None - self.weight = None - self.weight_backward = None + self.is_sparse = False + self.next_layer_repr = None self.build() def build(self): - self.adjacency_matrix = [] - self.adjacency_matrix_backward = [] - self.weight = [] - self.weight_backward = [] + self.next_layer_repr = torch.nn.ModuleList([ + torch.nn.ModuleList() \ + for _ in range(len(self.data.node_types)) + ]) for fam in self.data.relation_families: - adj_mat = [ rel.adjacency_matrix \ - for rel in fam.relation_types \ - if rel.adjacency_matrix is not None ] - adj_mat_back = [ rel.adjacency_matrix_backward \ - for rel in fam.relation_types \ - if rel.adjacency_matrix_backward is not None ] - weight = [ init_glorot(self.input_dim[fam.node_type_column], - self.output_dim[fam.node_type_row]) \ - for _ in range(len(adj_mat)) ] - weight_back = [ init_glorot(self.input_dim[fam.node_type_column], - self.output_dim[fam.node_type_row]) \ - for _ in range(len(adj_mat_back)) ] - adj_mat = torch.cat(adj_mat) \ - if len(adj_mat) > 0 \ - else None - adj_mat_back = torch.cat(adj_mat_back) \ - if len(adj_mat_back) > 0 \ - else None - self.adjacency_matrix.append(adj_mat) - self.adjacency_matrix_backward.append(adj_mat_back) - self.weight.append(weight) - self.weight_backward.append(weight_back) + self.build_family(fam) + + def build_family(self, fam) -> None: + if fam.node_type_row == fam.node_type_column: + self.build_fam_one_node_type(fam) + else: + self.build_fam_two_node_types(fam) + + def build_fam_one_node_type(self, fam) -> None: + adjacency_matrices = [ + r.adjacency_matrix \ + for r in fam.relation_types + ] + conv = FastGraphConv(self.input_dim[fam.node_type_column], + self.output_dim[fam.node_type_row], + adjacency_matrices, + self.keep_prob, + self.rel_activation) + conv.input_node_type = fam.node_type_column + self.next_layer_repr[fam.node_type_row].append(conv) + + def build_fam_two_node_types(self, fam) -> None: + adjacency_matrices = [ + r.adjacency_matrix \ + for r in fam.relation_types \ + if r.adjacency_matrix is not None + ] + + adjacency_matrices_backward = [ + r.adjacency_matrix_backward \ + for r in fam.relation_types \ + if r.adjacency_matrix_backward is not None + ] + + conv = FastGraphConv(self.input_dim[fam.node_type_column], + self.output_dim[fam.node_type_row], + adjacency_matrices, + self.keep_prob, + self.rel_activation) + + conv_backward = FastGraphConv(self.input_dim[fam.node_type_row], + self.output_dim[fam.node_type_column], + adjacency_matrices_backward, + self.keep_prob, + self.rel_activation) + + conv.input_node_type = fam.node_type_column + conv_backward.input_node_type = fam.node_type_row + + self.next_layer_repr[fam.node_type_row].append(conv) + self.next_layer_repr[fam.node_type_column].append(conv_backward) def forward(self, prev_layer_repr): - for i, fam in enumerate(self.data.relation_families): - repr_row = prev_layer_repr[fam.node_type_row] - repr_column = prev_layer_repr[fam.node_type_column] - - adj_mat = self.adjacency_matrix[i] - adj_mat_back = self.adjacency_matrix_backward[i] - - if adj_mat is not None: - x = dropout(repr_column, keep_prob=self.keep_prob) - x = torch.sparse.mm(x, self.weight[i]) \ - if x.is_sparse \ - else torch.mm(x, self.weight[i]) - x = torch.sparse.mm(adj_mat, x) \ - if adj_mat.is_sparse \ - else torch.mm(adj_mat, x) - x = self.rel_activation(x) - x = x.view(len(fam.relation_types), len(repr_row), -1) - - if adj_mat_back is not None: - x = dropout(repr_column, keep_prob=self.keep_prob) - x = torch.sparse.mm(x, self.weight_backward[i]) \ - if x.is_sparse \ - else torch.mm(x, self.weight_backward[i]) - x = torch.sparse.mm(adj_mat_back, x) \ - if adj_mat_back.is_sparse \ - else torch.mm(adj_mat_back, x) - x = self.rel_activation(x) - x = x.view(len(fam.relation_types), len(repr_row), -1) + next_layer_repr = [ [] \ + for _ in range(len(self.data.node_types)) ] + for output_node_type in range(len(self.data.node_types)): + for conv in self.next_layer_repr[output_node_type]: + rep = conv(prev_layer_repr[conv.input_node_type]) + rep = torch.sum(rep, dim=0) + rep = torch.nn.functional.normalize(rep, p=2, dim=1) + next_layer_repr[output_node_type].append(rep) + if len(next_layer_repr[output_node_type]) == 0: + next_layer_repr[output_node_type] = \ + torch.zeros(self.data.node_types[output_node_type].count, self.output_dim[output_node_type]) + else: + next_layer_repr[output_node_type] = \ + sum(next_layer_repr[output_node_type]) + next_layer_repr[output_node_type] = \ + self.layer_activation(next_layer_repr[output_node_type]) + return next_layer_repr @staticmethod def _check_params(input_dim, output_dim, data, keep_prob, -- 2.26.2 From b9b97f3dd7ade627d1320ee7b69342cc289ab9d4 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 28 Jul 2020 10:20:23 +0200 Subject: [PATCH 172/227] Add test_fast_conv_layer_01() and test_fast_conv_layer_02(). --- tests/icosagon/test_fastconv.py | 41 +++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/tests/icosagon/test_fastconv.py b/tests/icosagon/test_fastconv.py index 742173d..2003316 100644 --- a/tests/icosagon/test_fastconv.py +++ b/tests/icosagon/test_fastconv.py @@ -18,7 +18,7 @@ def _make_symmetric(x: torch.Tensor): def _symmetric_random(n_rows, n_columns): return _make_symmetric(torch.rand((n_rows, n_columns), - dtype=torch.float32).round()) + dtype=torch.float32).round().to_sparse()) def _some_data_with_interactions(): @@ -28,7 +28,7 @@ def _some_data_with_interactions(): fam = d.add_relation_family('Drug-Gene', 1, 0, True) fam.add_relation_type('Target', - torch.rand((100, 1000), dtype=torch.float32).round()) + torch.rand((100, 1000), dtype=torch.float32).round().to_sparse()) fam = d.add_relation_family('Gene-Gene', 0, 0, True) fam.add_relation_type('Interaction', @@ -164,10 +164,47 @@ def test_fast_conv_layer_01(): d = _some_data_with_interactions() in_layer = OneHotInputLayer(d) + d_layer = DecagonLayer(in_layer.output_dim, [32, 32], d) + seq_1 = torch.nn.Sequential(in_layer, d_layer) + _ = seq_1(None) + + conv_layer = FastConvLayer(in_layer.output_dim, [32, 32], d) + seq_2 = torch.nn.Sequential(in_layer, conv_layer) + _ = seq_2(None) + + +def test_fast_conv_layer_02(): + d = _some_data_with_interactions() + in_layer = OneHotInputLayer(d) + d_layer = DecagonLayer(in_layer.output_dim, [32, 32], d) seq_1 = torch.nn.Sequential(in_layer, d_layer) out_repr_1 = seq_1(None) + assert len(d_layer.next_layer_repr[0]) == 2 + assert len(d_layer.next_layer_repr[1]) == 2 + conv_layer = FastConvLayer(in_layer.output_dim, [32, 32], d) + assert len(conv_layer.next_layer_repr[1]) == 2 + conv_layer.next_layer_repr[1][0].weights = torch.cat([ + d_layer.next_layer_repr[1][0].convolutions[0].graph_conv.weight, + ], dim=1) + conv_layer.next_layer_repr[1][1].weights = torch.cat([ + d_layer.next_layer_repr[1][1].convolutions[0].graph_conv.weight, + d_layer.next_layer_repr[1][1].convolutions[1].graph_conv.weight, + d_layer.next_layer_repr[1][1].convolutions[2].graph_conv.weight, + ], dim=1) + assert len(conv_layer.next_layer_repr[0]) == 2 + conv_layer.next_layer_repr[0][0].weights = torch.cat([ + d_layer.next_layer_repr[0][0].convolutions[0].graph_conv.weight, + ], dim=1) + conv_layer.next_layer_repr[0][1].weights = torch.cat([ + d_layer.next_layer_repr[0][1].convolutions[0].graph_conv.weight, + ], dim=1) + seq_2 = torch.nn.Sequential(in_layer, conv_layer) out_repr_2 = seq_2(None) + + assert len(out_repr_1) == len(out_repr_2) + for i in range(len(out_repr_1)): + assert torch.all(out_repr_1[i] == out_repr_2[i]) -- 2.26.2 From f99f0bb9195d5a7fe54b820c20fe4664263f16d6 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 28 Jul 2020 11:39:34 +0200 Subject: [PATCH 173/227] Add FastModel. --- src/icosagon/fastmodel.py | 74 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 src/icosagon/fastmodel.py diff --git a/src/icosagon/fastmodel.py b/src/icosagon/fastmodel.py new file mode 100644 index 0000000..7c30906 --- /dev/null +++ b/src/icosagon/fastmodel.py @@ -0,0 +1,74 @@ +from .fastconv import FastConvLayer +from .bulkdec import BulkDecodeLayer +from .input import OneHotInputLayer +import torch +import types + + +class FastModel(torch.nn.Module): + def __init(self, prep_d: PreparedData, + layer_dimensions: List[int] = [32, 64], + keep_prob: float = 1., + rel_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, + layer_activation: Callable[[torch.Tensor], torch.Tensor] = torch.nn.functional.relu, + dec_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, + **kwargs) -> None: + + super().__init__(**kwargs) + + self._check_params(prep_d, layer_dimensions, rel_activation, + layer_activation, dec_activation) + + self.prep_d = prep_d + self.keep_prob = float(keep_prob) + self.rel_activation = rel_activation + self.layer_activation = layer_activation + self.dec_activation = dec_activation + + self.seq = None + self.build() + + def build(self): + in_layer = OneHotInputLayer(self.prep_d) + last_output_dim = in_layer.output_dim + seq = [ in_layer ] + + for dim in self.layer_dimensions: + conv_layer = FastConvLayer(input_dim = last_output_dim, + output_dim = [dim] * len(self.prep_d.node_types), + data = self.prep_d, + keep_prob = self.keep_prob, + rel_activation = self.rel_activation, + layer_activation = self.layer_activation) + last_output_dim = conv_layer.output_dim + seq.append(conv_layer) + + dec_layer = BulkDecodeLayer(input_dim = last_output_dim, + data = self.prep_d, + keep_prob = self.keep_prob, + activation = self.dec_activation) + seq.append(dec_layer) + + seq = torch.nn.Sequential(*seq) + self.seq = seq + + def forward(self, _): + return self.seq(None) + + def self._check_params(self, prep_d, layer_dimensions, rel_activation, + layer_activation, dec_activation): + + if not isinstance(prep_d, PreparedData): + raise TypeError('prep_d must be an instanced of PreparedData') + + if not isinstance(layer_dimensions, list): + raise TypeError('layer_dimensions must be a list') + + if not isinstance(rel_activation, types.FunctionType): + raise TypeError('rel_activation must be a function') + + if not isinstance(layer_activation, types.FunctionType): + raise TypeError('layer_activation must be a function') + + if not isinstance(dec_activation, types.FunctionType): + raise TypeError('dec_activation must be a function') -- 2.26.2 From 8edf8ce4f99d0df92b919d19957d1758665cdeaa Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 28 Jul 2020 12:17:00 +0200 Subject: [PATCH 174/227] Add test_fast_model_01(). --- src/icosagon/fastmodel.py | 9 ++++-- tests/icosagon/test_fastmodel.py | 50 ++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 tests/icosagon/test_fastmodel.py diff --git a/src/icosagon/fastmodel.py b/src/icosagon/fastmodel.py index 7c30906..a68fe58 100644 --- a/src/icosagon/fastmodel.py +++ b/src/icosagon/fastmodel.py @@ -1,12 +1,16 @@ from .fastconv import FastConvLayer from .bulkdec import BulkDecodeLayer from .input import OneHotInputLayer +from .trainprep import PreparedData import torch import types +from typing import List, \ + Union, \ + Callable class FastModel(torch.nn.Module): - def __init(self, prep_d: PreparedData, + def __init__(self, prep_d: PreparedData, layer_dimensions: List[int] = [32, 64], keep_prob: float = 1., rel_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, @@ -20,6 +24,7 @@ class FastModel(torch.nn.Module): layer_activation, dec_activation) self.prep_d = prep_d + self.layer_dimensions = layer_dimensions self.keep_prob = float(keep_prob) self.rel_activation = rel_activation self.layer_activation = layer_activation @@ -55,7 +60,7 @@ class FastModel(torch.nn.Module): def forward(self, _): return self.seq(None) - def self._check_params(self, prep_d, layer_dimensions, rel_activation, + def _check_params(self, prep_d, layer_dimensions, rel_activation, layer_activation, dec_activation): if not isinstance(prep_d, PreparedData): diff --git a/tests/icosagon/test_fastmodel.py b/tests/icosagon/test_fastmodel.py new file mode 100644 index 0000000..6d5948a --- /dev/null +++ b/tests/icosagon/test_fastmodel.py @@ -0,0 +1,50 @@ +from icosagon.fastmodel import FastModel +from icosagon.data import Data +from icosagon.trainprep import prepare_training, \ + TrainValTest +import torch +import time + + +def _make_symmetric(x: torch.Tensor): + x = (x + x.transpose(0, 1)) / 2 + return x + + +def _symmetric_random(n_rows, n_columns): + return _make_symmetric(torch.rand((n_rows, n_columns), + dtype=torch.float32).round().to_sparse()) + + +def _some_data_with_interactions(): + d = Data() + d.add_node_type('Gene', 1000) + d.add_node_type('Drug', 100) + + fam = d.add_relation_family('Drug-Gene', 1, 0, True) + fam.add_relation_type('Target', + torch.rand((100, 1000), dtype=torch.float32).round().to_sparse()) + + fam = d.add_relation_family('Gene-Gene', 0, 0, True) + fam.add_relation_type('Interaction', + _symmetric_random(1000, 1000)) + + fam = d.add_relation_family('Drug-Drug', 1, 1, True) + for i in range(500): + fam.add_relation_type('Side Effect: Nausea %d' % i, + _symmetric_random(100, 100)) + fam.add_relation_type('Side Effect: Infertility %d' % i, + _symmetric_random(100, 100)) + fam.add_relation_type('Side Effect: Death %d' % i, + _symmetric_random(100, 100)) + return d + + +def test_fast_model_01(): + d = _some_data_with_interactions() + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) + model = FastModel(prep_d) + for i in range(10): + t = time.time() + _ = model(None) + print('Model forward took:', time.time() - t) -- 2.26.2 From 2eff854467457ee74a57f3fe233d0c76ae4b238e Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 30 Jul 2020 12:46:20 +0200 Subject: [PATCH 175/227] Start working on FastLoop. --- src/icosagon/fastloop.py | 166 ++++++++++++++++++++++++++++++++ tests/icosagon/test_fastloop.py | 51 ++++++++++ 2 files changed, 217 insertions(+) create mode 100644 src/icosagon/fastloop.py create mode 100644 tests/icosagon/test_fastloop.py diff --git a/src/icosagon/fastloop.py b/src/icosagon/fastloop.py new file mode 100644 index 0000000..f955932 --- /dev/null +++ b/src/icosagon/fastloop.py @@ -0,0 +1,166 @@ +from .fastmodel import FastModel +from .trainprep import PreparedData +import torch +from typing import Callable +from types import FunctionType +import time +import random + + +class FastBatcher(object): + def __init__(self, prep_d: PreparedData, batch_size: int, + shuffle: bool, generator: torch.Generator, + part_type: str) -> None: + + if not isinstance(prep_d, PreparedData): + raise TypeError('prep_d must be an instance of PreparedData') + + if not isinstance(generator, torch.Generator): + raise TypeError('generator must be an instance of torch.Generator') + + if part_type not in ['train', 'val', 'test']: + raise ValueError('part_type must be set to train, val or test') + + self.prep_d = prep_d + self.batch_size = int(batch_size) + self.shuffle = bool(shuffle) + self.generator = generator + self.part_type = part_type + + self.edges = None + self.targets = None + self.build() + + def build(self): + self.edges = [] + self.targets = [] + + for fam in self.prep_d.relation_families: + edges = [] + targets = [] + for i, rel in enumerate(fam.relation_types): + + edges_pos = getattr(rel.edges_pos, self.part_type) + edges_neg = getattr(rel.edges_neg, self.part_type) + edges_back_pos = getattr(rel.edges_back_pos, self.part_type) + edges_back_neg = getattr(rel.edges_back_neg, self.part_type) + + e = torch.cat([ edges_pos, + torch.cat([edges_back_pos[:, 1], edges_back_pos[:, 0]], dim=1) ]) + e = torch.cat([torch.ones(len(e), 1, dtype=torch.long) * i , e ], dim=1) + t = torch.ones(len(e)) + edges.append(e) + targets.append(t) + + e = torch.cat([ edges_neg, + torch.cat([edges_back_neg[:, 1], edges_back_neg[:, 0]], dim=1) ]) + e = torch.cat([ torch.ones(len(e), 1, dtype=torch.long) * i, e ], dim=1) + t = torch.zeros(len(e)) + edges.append(e) + targets.append(t) + + edges = torch.cat(edges) + targets = torch.cat(targets) + + self.edges.append(edges) + self.targets.append(targets) + + # print(self.edges) + # print(self.targets) + + if self.shuffle: + self.shuffle_families() + + def shuffle_families(self): + for i in range(len(self.edges)): + edges = self.edges[i] + targets = self.targets[i] + order = torch.randperm(len(edges), generator=self.generator) + self.edges[i] = edges[order] + self.targets[i] = targets[order] + + def __iter__(self): + offsets = [ 0 for _ in self.edges ] + + while True: + choice = [ i for i in range(len(offsets)) \ + if offsets[i] < len(self.edges[i]) ] + if len(choice) == 0: + break + fam_idx = torch.randint(len(choice), (1,), generator=self.generator).item() + ofs = offsets[fam_idx] + edges = self.edges[fam_idx][ofs:ofs + self.batch_size] + targets = self.targets[fam_idx][ofs:ofs + self.batch_size] + offsets[fam_idx] += self.batch_size + yield (fam_idx, edges, targets) + + +class FastLoop(object): + def __init__( + self, + model: FastModel, + lr: float = 0.001, + loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = \ + torch.nn.functional.binary_cross_entropy_with_logits, + batch_size: int = 100, + shuffle: bool = True, + generator: torch.Generator = None) -> None: + + self._check_params(model, loss, generator) + + self.model = model + self.lr = float(lr) + self.loss = loss + self.batch_size = int(batch_size) + self.shuffle = bool(shuffle) + self.generator = generator or torch.default_generator + + self.opt = None + + self.build() + + def _check_params(self, model, loss, generator): + if not isinstance(model, FastModel): + raise TypeError('model must be an instance of FastModel') + + if not isinstance(loss, FunctionType): + raise TypeError('loss must be a function') + + if generator is not None and not isinstance(generator, torch.Generator): + raise TypeError('generator must be an instance of torch.Generator') + + def build(self) -> None: + opt = torch.optim.Adam(self.model.parameters(), lr=self.lr) + self.opt = opt + + def run_epoch(self): + prep_d = self.model.prep_d + + batcher = FastBatcher(self.model.prep_d, batch_size=self.batch_size, + shuffle = self.shuffle, generator=self.generator) + # pred = self.model(None) + # n = len(list(iter(batch))) + loss_sum = 0 + for fam_idx, edges, targets in batcher: + self.opt.zero_grad() + pred = self.model(None) + + # process pred, get input and targets + input = pred[fam_idx][edges[:, 0], edges[:, 1]] + + loss = self.loss(input, targets) + loss.backward() + self.opt.step() + loss_sum += loss.detach().cpu().item() + return loss_sum + + + def train(self, max_epochs): + best_loss = None + best_epoch = None + for i in range(max_epochs): + loss = self.run_epoch() + if best_loss is None or loss < best_loss: + best_loss = loss + best_epoch = i + return loss, best_loss, best_epoch diff --git a/tests/icosagon/test_fastloop.py b/tests/icosagon/test_fastloop.py new file mode 100644 index 0000000..0afb285 --- /dev/null +++ b/tests/icosagon/test_fastloop.py @@ -0,0 +1,51 @@ +from icosagon.fastloop import FastBatcher, \ + FastModel +from icosagon.data import Data +from icosagon.trainprep import prepare_training, \ + TrainValTest +import torch + + +def test_fast_batcher_01(): + d = Data() + d.add_node_type('Gene', 5) + d.add_node_type('Drug', 3) + + fam = d.add_relation_family('Gene-Drug', 0, 1, True) + + adj_mat = torch.tensor([ + [ 1, 0, 1 ], + [ 0, 0, 1 ], + [ 0, 1, 0 ], + [ 1, 0, 0 ], + [ 0, 1, 1 ] + ], dtype=torch.float32).to_sparse() + fam.add_relation_type('Target', adj_mat) + + prep_d = prepare_training(d, TrainValTest(.8, .1, .1)) + # print(prep_d.relation_families[0]) + + g = torch.Generator() + batcher = FastBatcher(prep_d, batch_size=3, shuffle=True, + generator=g, part_type='train') + + print(batcher.edges) + print(batcher.targets) + + edges_check = [ set() for _ in range(len(batcher.edges)) ] + + for fam_idx, edges, targets in batcher: + print(fam_idx, edges, targets) + for e in edges: + edges_check[fam_idx].add(tuple(e.tolist())) + + edges_check_2 = [ set() for _ in range(len(batcher.edges)) ] + for i, edges in enumerate(batcher.edges): + for e in edges: + edges_check_2[i].add(tuple(e.tolist())) + + assert edges_check == edges_check_2 + + +def test_fast_model_01(): + raise NotImplementedError -- 2.26.2 From d57af9c09001b3ddfe5ba7c0f6a09c0dc2496380 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 3 Aug 2020 19:12:28 +0200 Subject: [PATCH 176/227] Start triacontagon. --- src/triacontagon/data.py | 209 ++++++++++++++++++++++++++++ src/triacontagon/decode.py | 123 ++++++++++++++++ src/triacontagon/dropout.py | 42 ++++++ src/triacontagon/fastconv.py | 255 ++++++++++++++++++++++++++++++++++ src/triacontagon/fastdec.py | 138 ++++++++++++++++++ src/triacontagon/fastloop.py | 166 ++++++++++++++++++++++ src/triacontagon/fastmodel.py | 79 +++++++++++ src/triacontagon/input.py | 79 +++++++++++ src/triacontagon/normalize.py | 145 +++++++++++++++++++ src/triacontagon/sampling.py | 47 +++++++ src/triacontagon/trainprep.py | 215 ++++++++++++++++++++++++++++ src/triacontagon/weights.py | 19 +++ 12 files changed, 1517 insertions(+) create mode 100644 src/triacontagon/data.py create mode 100644 src/triacontagon/decode.py create mode 100644 src/triacontagon/dropout.py create mode 100644 src/triacontagon/fastconv.py create mode 100644 src/triacontagon/fastdec.py create mode 100644 src/triacontagon/fastloop.py create mode 100644 src/triacontagon/fastmodel.py create mode 100644 src/triacontagon/input.py create mode 100644 src/triacontagon/normalize.py create mode 100644 src/triacontagon/sampling.py create mode 100644 src/triacontagon/trainprep.py create mode 100644 src/triacontagon/weights.py diff --git a/src/triacontagon/data.py b/src/triacontagon/data.py new file mode 100644 index 0000000..4505adf --- /dev/null +++ b/src/triacontagon/data.py @@ -0,0 +1,209 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +from collections import defaultdict +from dataclasses import dataclass, field +import torch +from typing import List, \ + Dict, \ + Tuple, \ + Any, \ + Type +from .decode import DEDICOMDecoder, \ + BilinearDecoder +import numpy as np + + +def _equal(x: torch.Tensor, y: torch.Tensor): + if x.is_sparse ^ y.is_sparse: + raise ValueError('Cannot mix sparse and dense tensors') + + if not x.is_sparse: + return (x == y) + + return ((x - y).coalesce().values() == 0) + + +@dataclass +class NodeType(object): + name: str + count: int + + +@dataclass +class RelationTypeBase(object): + name: str + node_type_row: int + node_type_column: int + adjacency_matrix: torch.Tensor + adjacency_matrix_backward: torch.Tensor + + +@dataclass +class RelationType(RelationTypeBase): + pass + + +@dataclass +class RelationFamilyBase(object): + data: 'Data' + name: str + node_type_row: int + node_type_column: int + is_symmetric: bool + decoder_class: Type + + +@dataclass +class RelationFamily(RelationFamilyBase): + relation_types: List[RelationType] = None + + def __post_init__(self) -> None: + if not self.is_symmetric and \ + self.decoder_class != DEDICOMDecoder and \ + self.decoder_class != BilinearDecoder: + raise TypeError('Family is assymetric but the specified decoder_class supports symmetric relations only') + + self.relation_types = [] + + def add_relation_type(self, + name: str, adjacency_matrix: torch.Tensor, + adjacency_matrix_backward: torch.Tensor = None) -> None: + + name = str(name) + node_type_row = self.node_type_row + node_type_column = self.node_type_column + + if adjacency_matrix is None and adjacency_matrix_backward is None: + raise ValueError('adjacency_matrix and adjacency_matrix_backward cannot both be None') + + if adjacency_matrix is not None and \ + not isinstance(adjacency_matrix, torch.Tensor): + raise ValueError('adjacency_matrix must be a torch.Tensor') + + if adjacency_matrix_backward is not None \ + and not isinstance(adjacency_matrix_backward, torch.Tensor): + raise ValueError('adjacency_matrix_backward must be a torch.Tensor') + + if adjacency_matrix is not None and \ + adjacency_matrix.shape != (self.data.node_types[node_type_row].count, + self.data.node_types[node_type_column].count): + raise ValueError('adjacency_matrix shape must be (num_row_nodes, num_column_nodes)') + + if adjacency_matrix_backward is not None and \ + adjacency_matrix_backward.shape != (self.data.node_types[node_type_column].count, + self.data.node_types[node_type_row].count): + raise ValueError('adjacency_matrix_backward shape must be (num_column_nodes, num_row_nodes)') + + if node_type_row == node_type_column and \ + adjacency_matrix_backward is not None: + raise ValueError('Relation between nodes of the same type must be expressed using a single matrix') + + if self.is_symmetric and adjacency_matrix_backward is not None: + raise ValueError('Cannot use a custom adjacency_matrix_backward in a symmetric relation family') + + if self.is_symmetric and node_type_row == node_type_column and \ + not torch.all(_equal(adjacency_matrix, + adjacency_matrix.transpose(0, 1))): + raise ValueError('Relation family is symmetric but adjacency_matrix is assymetric') + + if not self.is_symmetric and node_type_row != node_type_column and \ + adjacency_matrix_backward is None: + raise ValueError('Relation is asymmetric but adjacency_matrix_backward is None') + + if self.is_symmetric and node_type_row != node_type_column: + adjacency_matrix_backward = adjacency_matrix.transpose(0, 1) + + self.relation_types.append(RelationType(name, + node_type_row, node_type_column, + adjacency_matrix, adjacency_matrix_backward)) + + def node_name(self, index): + return self.data.node_types[index].name + + def __repr__(self): + s = 'Relation family %s' % self.name + + for r in self.relation_types: + s += '\n - %s%s' % (r.name, ' (two-way)' \ + if (r.adjacency_matrix is not None \ + and r.adjacency_matrix_backward is not None) \ + or self.node_type_row == self.node_type_column \ + else '%s <- %s' % (self.node_name(self.node_type_row), + self.node_name(self.node_type_column))) + + return s + + def repr_indented(self): + s = ' - %s' % self.name + + for r in self.relation_types: + s += '\n - %s%s' % (r.name, ' (two-way)' \ + if (r.adjacency_matrix is not None \ + and r.adjacency_matrix_backward is not None) \ + or self.node_type_row == self.node_type_column \ + else '%s <- %s' % (self.node_name(self.node_type_row), + self.node_name(self.node_type_column))) + + return s + + +class Data(object): + node_types: List[NodeType] + relation_families: List[RelationFamily] + + def __init__(self) -> None: + self.node_types = [] + self.relation_families = [] + + def add_node_type(self, name: str, count: int) -> None: + name = str(name) + count = int(count) + if not name: + raise ValueError('You must provide a non-empty node type name') + if count <= 0: + raise ValueError('You must provide a positive node count') + self.node_types.append(NodeType(name, count)) + + def add_relation_family(self, name: str, node_type_row: int, + node_type_column: int, is_symmetric: bool, + decoder_class: Type = DEDICOMDecoder): + + name = str(name) + node_type_row = int(node_type_row) + node_type_column = int(node_type_column) + is_symmetric = bool(is_symmetric) + + if node_type_row < 0 or node_type_row >= len(self.node_types): + raise ValueError('node_type_row outside of the valid range of node types') + + if node_type_column < 0 or node_type_column >= len(self.node_types): + raise ValueError('node_type_column outside of the valid range of node types') + + fam = RelationFamily(self, name, node_type_row, node_type_column, + is_symmetric, decoder_class) + self.relation_families.append(fam) + + return fam + + def __repr__(self): + n = len(self.node_types) + if n == 0: + return 'Empty Icosagon Data' + s = '' + s += 'Icosagon Data with:\n' + s += '- ' + str(n) + ' node type(s):\n' + for nt in self.node_types: + s += ' - ' + nt.name + '\n' + if len(self.relation_families) == 0: + s += '- No relation families\n' + return s.strip() + + s += '- %d relation families:\n' % len(self.relation_families) + for fam in self.relation_families: + s += fam.repr_indented() + '\n' + + return s.strip() diff --git a/src/triacontagon/decode.py b/src/triacontagon/decode.py new file mode 100644 index 0000000..00df8b2 --- /dev/null +++ b/src/triacontagon/decode.py @@ -0,0 +1,123 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import torch +from .weights import init_glorot +from .dropout import dropout + + +class DEDICOMDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, keep_prob=1., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.keep_prob = keep_prob + self.activation = activation + + self.global_interaction = torch.nn.Parameter(init_glorot(input_dim, input_dim)) + self.local_variation = torch.nn.ParameterList([ + torch.nn.Parameter(torch.flatten(init_glorot(input_dim, 1))) \ + for _ in range(num_relation_types) + ]) + + def forward(self, inputs_row, inputs_col, relation_index): + inputs_row = dropout(inputs_row, self.keep_prob) + inputs_col = dropout(inputs_col, self.keep_prob) + + relation = torch.diag(self.local_variation[relation_index]) + + product1 = torch.mm(inputs_row, relation) + product2 = torch.mm(product1, self.global_interaction) + product3 = torch.mm(product2, relation) + rec = torch.bmm(product3.view(product3.shape[0], 1, product3.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) + + return self.activation(rec) + + +class DistMultDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, keep_prob=1., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.keep_prob = keep_prob + self.activation = activation + + self.relation = torch.nn.ParameterList([ + torch.nn.Parameter(torch.flatten(init_glorot(input_dim, 1))) \ + for _ in range(num_relation_types) + ]) + + def forward(self, inputs_row, inputs_col, relation_index): + inputs_row = dropout(inputs_row, self.keep_prob) + inputs_col = dropout(inputs_col, self.keep_prob) + + relation = torch.diag(self.relation[relation_index]) + + intermediate_product = torch.mm(inputs_row, relation) + rec = torch.bmm(intermediate_product.view(intermediate_product.shape[0], 1, intermediate_product.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) + + return self.activation(rec) + + +class BilinearDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, keep_prob=1., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.keep_prob = keep_prob + self.activation = activation + + self.relation = torch.nn.ParameterList([ + torch.nn.Parameter(init_glorot(input_dim, input_dim)) \ + for _ in range(num_relation_types) + ]) + + def forward(self, inputs_row, inputs_col, relation_index): + inputs_row = dropout(inputs_row, self.keep_prob) + inputs_col = dropout(inputs_col, self.keep_prob) + + intermediate_product = torch.mm(inputs_row, self.relation[relation_index]) + rec = torch.bmm(intermediate_product.view(intermediate_product.shape[0], 1, intermediate_product.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) + + return self.activation(rec) + + +class InnerProductDecoder(torch.nn.Module): + """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" + def __init__(self, input_dim, num_relation_types, keep_prob=1., + activation=torch.sigmoid, **kwargs): + + super().__init__(**kwargs) + self.input_dim = input_dim + self.num_relation_types = num_relation_types + self.keep_prob = keep_prob + self.activation = activation + + + def forward(self, inputs_row, inputs_col, _): + inputs_row = dropout(inputs_row, self.keep_prob) + inputs_col = dropout(inputs_col, self.keep_prob) + + rec = torch.bmm(inputs_row.view(inputs_row.shape[0], 1, inputs_row.shape[1]), + inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) + rec = torch.flatten(rec) + + return self.activation(rec) diff --git a/src/triacontagon/dropout.py b/src/triacontagon/dropout.py new file mode 100644 index 0000000..63cfb58 --- /dev/null +++ b/src/triacontagon/dropout.py @@ -0,0 +1,42 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import torch +from .normalize import _sparse_coo_tensor + + +def dropout_sparse(x, keep_prob): + x = x.coalesce() + i = x._indices() + v = x._values() + size = x.size() + + n = keep_prob + torch.rand(len(v)) + n = torch.floor(n).to(torch.bool) + i = i[:,n] + v = v[n] + x = _sparse_coo_tensor(i, v, size=size) + + return x * (1./keep_prob) + + +def dropout_dense(x, keep_prob): + # print('dropout_dense()') + x = x.clone() + i = torch.nonzero(x) + + n = keep_prob + torch.rand(len(i)) + n = (1. - torch.floor(n)).to(torch.bool) + x[i[n, 0], i[n, 1]] = 0. + + return x * (1./keep_prob) + + +def dropout(x, keep_prob): + if x.is_sparse: + return dropout_sparse(x, keep_prob) + else: + return dropout_dense(x, keep_prob) diff --git a/src/triacontagon/fastconv.py b/src/triacontagon/fastconv.py new file mode 100644 index 0000000..038e2fc --- /dev/null +++ b/src/triacontagon/fastconv.py @@ -0,0 +1,255 @@ +from typing import List, \ + Union, \ + Callable +from .data import Data, \ + RelationFamily +from .trainprep import PreparedData, \ + PreparedRelationFamily +import torch +from .weights import init_glorot +from .normalize import _sparse_coo_tensor +import types + + +def _sparse_diag_cat(matrices: List[torch.Tensor]): + if len(matrices) == 0: + raise ValueError('The list of matrices must be non-empty') + + if not all(m.is_sparse for m in matrices): + raise ValueError('All matrices must be sparse') + + if not all(len(m.shape) == 2 for m in matrices): + raise ValueError('All matrices must be 2D') + + indices = [] + values = [] + row_offset = 0 + col_offset = 0 + + for m in matrices: + ind = m._indices().clone() + ind[0] += row_offset + ind[1] += col_offset + indices.append(ind) + values.append(m._values()) + row_offset += m.shape[0] + col_offset += m.shape[1] + + indices = torch.cat(indices, dim=1) + values = torch.cat(values) + + return _sparse_coo_tensor(indices, values, size=(row_offset, col_offset)) + + +def _cat(matrices: List[torch.Tensor]): + if len(matrices) == 0: + raise ValueError('Empty list passed to _cat()') + + n = sum(a.is_sparse for a in matrices) + if n != 0 and n != len(matrices): + raise ValueError('All matrices must have the same layout (dense or sparse)') + + if not all(a.shape[1:] == matrices[0].shape[1:] for a in matrices): + raise ValueError('All matrices must have the same dimensions apart from dimension 0') + + if not matrices[0].is_sparse: + return torch.cat(matrices) + + total_rows = sum(a.shape[0] for a in matrices) + indices = [] + values = [] + row_offset = 0 + + for a in matrices: + ind = a._indices().clone() + val = a._values() + ind[0] += row_offset + ind = ind.transpose(0, 1) + indices.append(ind) + values.append(val) + row_offset += a.shape[0] + + indices = torch.cat(indices).transpose(0, 1) + values = torch.cat(values) + + res = _sparse_coo_tensor(indices, values, size=(row_offset, matrices[0].shape[1])) + return res + + +class FastGraphConv(torch.nn.Module): + def __init__(self, + in_channels: int, + out_channels: int, + adjacency_matrices: List[torch.Tensor], + keep_prob: float = 1., + activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, + **kwargs) -> None: + + super().__init__(**kwargs) + + in_channels = int(in_channels) + out_channels = int(out_channels) + if not isinstance(adjacency_matrices, list): + raise TypeError('adjacency_matrices must be a list') + if len(adjacency_matrices) == 0: + raise ValueError('adjacency_matrices must not be empty') + if not all(isinstance(m, torch.Tensor) for m in adjacency_matrices): + raise TypeError('adjacency_matrices elements must be of class torch.Tensor') + if not all(m.is_sparse for m in adjacency_matrices): + raise ValueError('adjacency_matrices elements must be sparse') + keep_prob = float(keep_prob) + if not isinstance(activation, types.FunctionType): + raise TypeError('activation must be a function') + + self.in_channels = in_channels + self.out_channels = out_channels + self.adjacency_matrices = adjacency_matrices + self.keep_prob = keep_prob + self.activation = activation + + self.num_row_nodes = len(adjacency_matrices[0]) + self.num_relation_types = len(adjacency_matrices) + + self.adjacency_matrices = _sparse_diag_cat(adjacency_matrices) + + self.weights = torch.cat([ + init_glorot(in_channels, out_channels) \ + for _ in range(self.num_relation_types) + ], dim=1) + + def forward(self, x) -> torch.Tensor: + if self.keep_prob < 1.: + x = dropout(x, self.keep_prob) + res = torch.sparse.mm(x, self.weights) \ + if x.is_sparse \ + else torch.mm(x, self.weights) + res = torch.split(res, res.shape[1] // self.num_relation_types, dim=1) + res = torch.cat(res) + res = torch.sparse.mm(self.adjacency_matrices, res) \ + if self.adjacency_matrices.is_sparse \ + else torch.mm(self.adjacency_matrices, res) + res = res.view(self.num_relation_types, self.num_row_nodes, self.out_channels) + if self.activation is not None: + res = self.activation(res) + + return res + + +class FastConvLayer(torch.nn.Module): + def __init__(self, + input_dim: List[int], + output_dim: List[int], + data: Union[Data, PreparedData], + keep_prob: float = 1., + rel_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, + layer_activation: Callable[[torch.Tensor], torch.Tensor] = torch.nn.functional.relu, + **kwargs): + + super().__init__(**kwargs) + + self._check_params(input_dim, output_dim, data, keep_prob, + rel_activation, layer_activation) + + self.input_dim = input_dim + self.output_dim = output_dim + self.data = data + self.keep_prob = keep_prob + self.rel_activation = rel_activation + self.layer_activation = layer_activation + + self.is_sparse = False + self.next_layer_repr = None + self.build() + + def build(self): + self.next_layer_repr = torch.nn.ModuleList([ + torch.nn.ModuleList() \ + for _ in range(len(self.data.node_types)) + ]) + for fam in self.data.relation_families: + self.build_family(fam) + + def build_family(self, fam) -> None: + if fam.node_type_row == fam.node_type_column: + self.build_fam_one_node_type(fam) + else: + self.build_fam_two_node_types(fam) + + def build_fam_one_node_type(self, fam) -> None: + adjacency_matrices = [ + r.adjacency_matrix \ + for r in fam.relation_types + ] + conv = FastGraphConv(self.input_dim[fam.node_type_column], + self.output_dim[fam.node_type_row], + adjacency_matrices, + self.keep_prob, + self.rel_activation) + conv.input_node_type = fam.node_type_column + self.next_layer_repr[fam.node_type_row].append(conv) + + def build_fam_two_node_types(self, fam) -> None: + adjacency_matrices = [ + r.adjacency_matrix \ + for r in fam.relation_types \ + if r.adjacency_matrix is not None + ] + + adjacency_matrices_backward = [ + r.adjacency_matrix_backward \ + for r in fam.relation_types \ + if r.adjacency_matrix_backward is not None + ] + + conv = FastGraphConv(self.input_dim[fam.node_type_column], + self.output_dim[fam.node_type_row], + adjacency_matrices, + self.keep_prob, + self.rel_activation) + + conv_backward = FastGraphConv(self.input_dim[fam.node_type_row], + self.output_dim[fam.node_type_column], + adjacency_matrices_backward, + self.keep_prob, + self.rel_activation) + + conv.input_node_type = fam.node_type_column + conv_backward.input_node_type = fam.node_type_row + + self.next_layer_repr[fam.node_type_row].append(conv) + self.next_layer_repr[fam.node_type_column].append(conv_backward) + + def forward(self, prev_layer_repr): + next_layer_repr = [ [] \ + for _ in range(len(self.data.node_types)) ] + for output_node_type in range(len(self.data.node_types)): + for conv in self.next_layer_repr[output_node_type]: + rep = conv(prev_layer_repr[conv.input_node_type]) + rep = torch.sum(rep, dim=0) + rep = torch.nn.functional.normalize(rep, p=2, dim=1) + next_layer_repr[output_node_type].append(rep) + if len(next_layer_repr[output_node_type]) == 0: + next_layer_repr[output_node_type] = \ + torch.zeros(self.data.node_types[output_node_type].count, self.output_dim[output_node_type]) + else: + next_layer_repr[output_node_type] = \ + sum(next_layer_repr[output_node_type]) + next_layer_repr[output_node_type] = \ + self.layer_activation(next_layer_repr[output_node_type]) + return next_layer_repr + + @staticmethod + def _check_params(input_dim, output_dim, data, keep_prob, + rel_activation, layer_activation): + + if not isinstance(input_dim, list): + raise ValueError('input_dim must be a list') + + if not output_dim: + raise ValueError('output_dim must be specified') + + if not isinstance(output_dim, list): + output_dim = [output_dim] * len(data.node_types) + + if not isinstance(data, Data) and not isinstance(data, PreparedData): + raise ValueError('data must be of type Data or PreparedData') diff --git a/src/triacontagon/fastdec.py b/src/triacontagon/fastdec.py new file mode 100644 index 0000000..ca08892 --- /dev/null +++ b/src/triacontagon/fastdec.py @@ -0,0 +1,138 @@ +import torch +from typing import List +from .trainprep import PreparedData +from dataclasses import dataclass +import random +from collections import defaultdict + + +@dataclass +class TrainingBatch(object): + relation_family_index: int + relation_type_index: int + node_type_row: int + node_type_column: int + edges: torch.Tensor + + +class FastBatcher(object): + def __init__(self, + prep_d: PreparedData, + batch_size: int) -> None: + + if not isinstance(prep_d, PreparedData): + raise TypeError('prep_d must be an instance of PreparedData') + + self.prep_d = prep_d + self.batch_size = int(batch_size) + + self.edges = None + self.build() + + def build(self): + self.edges = [] + for fam_idx, fam in enumerate(self.prep_d.relation_families): + edges = [] + targets = [] + edges_back = [] + targets_back = [] + for rel_idx, rel in enumerate(fam.relation_types): + edges.append(rel.edges_pos.train) + edges.append(rel.edges_neg.train) + targets.append(torch.ones(len(rel.edges_pos.train))) + targets.append(torch.zeros(len(rel.edges_neg.train))) + + edges_back.append(rel.edges_back_pos.train) + edges_back.append(rel.edges_back_neg.train) + targets_back.apend(torch.zeros(len(rel.edges_back_pos.train))) + targets_back.apend(torch.zeros(len(rel.edges_back_neg.train))) + + edges = torch.cat(edges) + targets = torch.cat(targets) + edges_back = torch.cat(edges_back) + targets_back = torch.cat(targets_back) + + order = torch.randperm(len(edges)) + edges = edges[order] + targets = targets[order] + + order_back = torch.randperm(len(edges_back)) + edges_back = edges_back[order_back] + targets_back = targets_back[order_back] + + self.edges.append({'fam_idx': fam_idx, 'rel_idx': rel_idx, 'back': False, + 'edges': edges, 'targets': targets, 'ofs': 0}) + self.edges.append({'fam_idx': fam_idx, 'rel_idx': rel_idx, 'back': True, + 'edges': edges_back, 'targets': targets_back, 'ofs': 0}) + + def __iter__(self): + while True: + edges = [ e for e in self.edges \ + if e['ofs'] < len(e['edges']) ] + # TODO: need to finish this + + def __iter_old__(self): + edge_types = ['edges_pos', 'edges_neg', 'edges_back_pos', 'edges_back_neg'] + + offsets = {} + orders = {} + done = {} + + for fam_idx, fam in enumerate(self.prep_d.relation_families): + for rel_idx, rel in enumerate(fam.relation_types): + for et in edge_types: + done[fam_idx, rel_idx, et] = False + + while True: + fam_idx = torch.randint(0, len(self.prep_d.relation_families), (1,)).item() + fam = self.prep_d.relation_families[fam_idx] + + rel_idx = torch.randint(0, len(fam.relation_types), (1,)).item() + rel = fam.relation_types[rel_idx] + + et = random.choice(edge_types) + edges = getattr(rel, et).train + + key = (fam_idx, rel_idx, et) + if key not in orders: + orders[key] = torch.randperm(len(edges)) + offsets[key] = 0 + + ord = orders[key] + ofs = offsets[key] + + nt_row = rel.node_type_row + nt_col = rel.node_type_column + + if 'back' in et: + nt_row, nt_col = nt_col, nt_row + + if ofs < len(edges): + offsets[key] += self.batch_size + ord = ord[ofs:ofs+self.batch_size] + edges = edges[ord] + yield TrainingBatch(fam_idx, rel_idx, nt_row, nt_column, edges) + else: + done[key] = True + + + + + for fam in self.prep_d.relation_families: + edges = [] + for rel in fam.relation_types: + edges.append(rel.edges_pos.train) + edges.append(rel.edges_back_pos.train) + edges.append(rel.edges_neg.train) + edges.append(rel.edges_back_neg.train) + edges = torch.cat(e) + + + +class FastDecLayer(torch.nn.Module): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def forward(self, + last_layer_repr: List[torch.Tensor], + training_batch: TrainingBatch): diff --git a/src/triacontagon/fastloop.py b/src/triacontagon/fastloop.py new file mode 100644 index 0000000..f955932 --- /dev/null +++ b/src/triacontagon/fastloop.py @@ -0,0 +1,166 @@ +from .fastmodel import FastModel +from .trainprep import PreparedData +import torch +from typing import Callable +from types import FunctionType +import time +import random + + +class FastBatcher(object): + def __init__(self, prep_d: PreparedData, batch_size: int, + shuffle: bool, generator: torch.Generator, + part_type: str) -> None: + + if not isinstance(prep_d, PreparedData): + raise TypeError('prep_d must be an instance of PreparedData') + + if not isinstance(generator, torch.Generator): + raise TypeError('generator must be an instance of torch.Generator') + + if part_type not in ['train', 'val', 'test']: + raise ValueError('part_type must be set to train, val or test') + + self.prep_d = prep_d + self.batch_size = int(batch_size) + self.shuffle = bool(shuffle) + self.generator = generator + self.part_type = part_type + + self.edges = None + self.targets = None + self.build() + + def build(self): + self.edges = [] + self.targets = [] + + for fam in self.prep_d.relation_families: + edges = [] + targets = [] + for i, rel in enumerate(fam.relation_types): + + edges_pos = getattr(rel.edges_pos, self.part_type) + edges_neg = getattr(rel.edges_neg, self.part_type) + edges_back_pos = getattr(rel.edges_back_pos, self.part_type) + edges_back_neg = getattr(rel.edges_back_neg, self.part_type) + + e = torch.cat([ edges_pos, + torch.cat([edges_back_pos[:, 1], edges_back_pos[:, 0]], dim=1) ]) + e = torch.cat([torch.ones(len(e), 1, dtype=torch.long) * i , e ], dim=1) + t = torch.ones(len(e)) + edges.append(e) + targets.append(t) + + e = torch.cat([ edges_neg, + torch.cat([edges_back_neg[:, 1], edges_back_neg[:, 0]], dim=1) ]) + e = torch.cat([ torch.ones(len(e), 1, dtype=torch.long) * i, e ], dim=1) + t = torch.zeros(len(e)) + edges.append(e) + targets.append(t) + + edges = torch.cat(edges) + targets = torch.cat(targets) + + self.edges.append(edges) + self.targets.append(targets) + + # print(self.edges) + # print(self.targets) + + if self.shuffle: + self.shuffle_families() + + def shuffle_families(self): + for i in range(len(self.edges)): + edges = self.edges[i] + targets = self.targets[i] + order = torch.randperm(len(edges), generator=self.generator) + self.edges[i] = edges[order] + self.targets[i] = targets[order] + + def __iter__(self): + offsets = [ 0 for _ in self.edges ] + + while True: + choice = [ i for i in range(len(offsets)) \ + if offsets[i] < len(self.edges[i]) ] + if len(choice) == 0: + break + fam_idx = torch.randint(len(choice), (1,), generator=self.generator).item() + ofs = offsets[fam_idx] + edges = self.edges[fam_idx][ofs:ofs + self.batch_size] + targets = self.targets[fam_idx][ofs:ofs + self.batch_size] + offsets[fam_idx] += self.batch_size + yield (fam_idx, edges, targets) + + +class FastLoop(object): + def __init__( + self, + model: FastModel, + lr: float = 0.001, + loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = \ + torch.nn.functional.binary_cross_entropy_with_logits, + batch_size: int = 100, + shuffle: bool = True, + generator: torch.Generator = None) -> None: + + self._check_params(model, loss, generator) + + self.model = model + self.lr = float(lr) + self.loss = loss + self.batch_size = int(batch_size) + self.shuffle = bool(shuffle) + self.generator = generator or torch.default_generator + + self.opt = None + + self.build() + + def _check_params(self, model, loss, generator): + if not isinstance(model, FastModel): + raise TypeError('model must be an instance of FastModel') + + if not isinstance(loss, FunctionType): + raise TypeError('loss must be a function') + + if generator is not None and not isinstance(generator, torch.Generator): + raise TypeError('generator must be an instance of torch.Generator') + + def build(self) -> None: + opt = torch.optim.Adam(self.model.parameters(), lr=self.lr) + self.opt = opt + + def run_epoch(self): + prep_d = self.model.prep_d + + batcher = FastBatcher(self.model.prep_d, batch_size=self.batch_size, + shuffle = self.shuffle, generator=self.generator) + # pred = self.model(None) + # n = len(list(iter(batch))) + loss_sum = 0 + for fam_idx, edges, targets in batcher: + self.opt.zero_grad() + pred = self.model(None) + + # process pred, get input and targets + input = pred[fam_idx][edges[:, 0], edges[:, 1]] + + loss = self.loss(input, targets) + loss.backward() + self.opt.step() + loss_sum += loss.detach().cpu().item() + return loss_sum + + + def train(self, max_epochs): + best_loss = None + best_epoch = None + for i in range(max_epochs): + loss = self.run_epoch() + if best_loss is None or loss < best_loss: + best_loss = loss + best_epoch = i + return loss, best_loss, best_epoch diff --git a/src/triacontagon/fastmodel.py b/src/triacontagon/fastmodel.py new file mode 100644 index 0000000..a68fe58 --- /dev/null +++ b/src/triacontagon/fastmodel.py @@ -0,0 +1,79 @@ +from .fastconv import FastConvLayer +from .bulkdec import BulkDecodeLayer +from .input import OneHotInputLayer +from .trainprep import PreparedData +import torch +import types +from typing import List, \ + Union, \ + Callable + + +class FastModel(torch.nn.Module): + def __init__(self, prep_d: PreparedData, + layer_dimensions: List[int] = [32, 64], + keep_prob: float = 1., + rel_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, + layer_activation: Callable[[torch.Tensor], torch.Tensor] = torch.nn.functional.relu, + dec_activation: Callable[[torch.Tensor], torch.Tensor] = lambda x: x, + **kwargs) -> None: + + super().__init__(**kwargs) + + self._check_params(prep_d, layer_dimensions, rel_activation, + layer_activation, dec_activation) + + self.prep_d = prep_d + self.layer_dimensions = layer_dimensions + self.keep_prob = float(keep_prob) + self.rel_activation = rel_activation + self.layer_activation = layer_activation + self.dec_activation = dec_activation + + self.seq = None + self.build() + + def build(self): + in_layer = OneHotInputLayer(self.prep_d) + last_output_dim = in_layer.output_dim + seq = [ in_layer ] + + for dim in self.layer_dimensions: + conv_layer = FastConvLayer(input_dim = last_output_dim, + output_dim = [dim] * len(self.prep_d.node_types), + data = self.prep_d, + keep_prob = self.keep_prob, + rel_activation = self.rel_activation, + layer_activation = self.layer_activation) + last_output_dim = conv_layer.output_dim + seq.append(conv_layer) + + dec_layer = BulkDecodeLayer(input_dim = last_output_dim, + data = self.prep_d, + keep_prob = self.keep_prob, + activation = self.dec_activation) + seq.append(dec_layer) + + seq = torch.nn.Sequential(*seq) + self.seq = seq + + def forward(self, _): + return self.seq(None) + + def _check_params(self, prep_d, layer_dimensions, rel_activation, + layer_activation, dec_activation): + + if not isinstance(prep_d, PreparedData): + raise TypeError('prep_d must be an instanced of PreparedData') + + if not isinstance(layer_dimensions, list): + raise TypeError('layer_dimensions must be a list') + + if not isinstance(rel_activation, types.FunctionType): + raise TypeError('rel_activation must be a function') + + if not isinstance(layer_activation, types.FunctionType): + raise TypeError('layer_activation must be a function') + + if not isinstance(dec_activation, types.FunctionType): + raise TypeError('dec_activation must be a function') diff --git a/src/triacontagon/input.py b/src/triacontagon/input.py new file mode 100644 index 0000000..3bf5824 --- /dev/null +++ b/src/triacontagon/input.py @@ -0,0 +1,79 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import torch +from typing import Union, \ + List +from .data import Data + + +class InputLayer(torch.nn.Module): + def __init__(self, data: Data, output_dim: Union[int, List[int]] = None, + **kwargs) -> None: + + output_dim = output_dim or \ + list(map(lambda a: a.count, data.node_types)) + + if not isinstance(output_dim, list): + output_dim = [output_dim,] * len(data.node_types) + + super().__init__(**kwargs) + self.output_dim = output_dim + self.data = data + + self.is_sparse=False + self.node_reps = None + self.build() + + def build(self) -> None: + self.node_reps = [] + for i, nt in enumerate(self.data.node_types): + reps = torch.rand(nt.count, self.output_dim[i]) + reps = torch.nn.Parameter(reps) + self.register_parameter('node_reps[%d]' % i, reps) + self.node_reps.append(reps) + + def forward(self, x) -> List[torch.nn.Parameter]: + return self.node_reps + + def __repr__(self) -> str: + s = '' + s += 'Icosagon input layer with output_dim: %s\n' % self.output_dim + s += ' # of node types: %d\n' % len(self.data.node_types) + for nt in self.data.node_types: + s += ' - %s (%d)\n' % (nt.name, nt.count) + return s.strip() + + +class OneHotInputLayer(torch.nn.Module): + def __init__(self, data: Data, **kwargs) -> None: + output_dim = [ a.count for a in data.node_types ] + super().__init__(**kwargs) + self.output_dim = output_dim + self.data = data + + self.is_sparse=True + self.node_reps = None + self.build() + + def build(self) -> None: + self.node_reps = torch.nn.ParameterList() + for i, nt in enumerate(self.data.node_types): + reps = torch.eye(nt.count).to_sparse() + reps = torch.nn.Parameter(reps, requires_grad=False) + # self.register_parameter('node_reps[%d]' % i, reps) + self.node_reps.append(reps) + + def forward(self, x) -> List[torch.nn.Parameter]: + return self.node_reps + + def __repr__(self) -> str: + s = '' + s += 'Icosagon one-hot input layer\n' + s += ' # of node types: %d\n' % len(self.data.node_types) + for nt in self.data.node_types: + s += ' - %s (%d)\n' % (nt.name, nt.count) + return s.strip() diff --git a/src/triacontagon/normalize.py b/src/triacontagon/normalize.py new file mode 100644 index 0000000..e13fb05 --- /dev/null +++ b/src/triacontagon/normalize.py @@ -0,0 +1,145 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import numpy as np +import scipy.sparse as sp +import torch + + +def _check_tensor(adj_mat): + if not isinstance(adj_mat, torch.Tensor): + raise ValueError('adj_mat must be a torch.Tensor') + + +def _check_sparse(adj_mat): + if not adj_mat.is_sparse: + raise ValueError('adj_mat must be sparse') + + +def _check_dense(adj_mat): + if adj_mat.is_sparse: + raise ValueError('adj_mat must be dense') + + +def _check_square(adj_mat): + if len(adj_mat.shape) != 2 or \ + adj_mat.shape[0] != adj_mat.shape[1]: + raise ValueError('adj_mat must be a square matrix') + + +def _check_2d(adj_mat): + if len(adj_mat.shape) != 2: + raise ValueError('adj_mat must be a square matrix') + + +def _sparse_coo_tensor(indices, values, size): + ctor = { torch.float32: torch.sparse.FloatTensor, + torch.float32: torch.sparse.DoubleTensor, + torch.uint8: torch.sparse.ByteTensor, + torch.long: torch.sparse.LongTensor, + torch.int: torch.sparse.IntTensor, + torch.short: torch.sparse.ShortTensor, + torch.bool: torch.sparse.ByteTensor }[values.dtype] + return ctor(indices, values, size) + + +def add_eye_sparse(adj_mat: torch.Tensor) -> torch.Tensor: + _check_tensor(adj_mat) + _check_sparse(adj_mat) + _check_square(adj_mat) + + adj_mat = adj_mat.coalesce() + indices = adj_mat.indices() + values = adj_mat.values() + + eye_indices = torch.arange(adj_mat.shape[0], dtype=indices.dtype, + device=adj_mat.device).view(1, -1) + eye_indices = torch.cat((eye_indices, eye_indices), 0) + eye_values = torch.ones(adj_mat.shape[0], dtype=values.dtype, + device=adj_mat.device) + + indices = torch.cat((indices, eye_indices), 1) + values = torch.cat((values, eye_values), 0) + + adj_mat = _sparse_coo_tensor(indices, values, adj_mat.shape) + + return adj_mat + + +def norm_adj_mat_one_node_type_sparse(adj_mat: torch.Tensor) -> torch.Tensor: + _check_tensor(adj_mat) + _check_sparse(adj_mat) + _check_square(adj_mat) + + adj_mat = add_eye_sparse(adj_mat) + adj_mat = norm_adj_mat_two_node_types_sparse(adj_mat) + + return adj_mat + + +def norm_adj_mat_one_node_type_dense(adj_mat: torch.Tensor) -> torch.Tensor: + _check_tensor(adj_mat) + _check_dense(adj_mat) + _check_square(adj_mat) + + adj_mat = adj_mat + torch.eye(adj_mat.shape[0], dtype=adj_mat.dtype, + device=adj_mat.device) + adj_mat = norm_adj_mat_two_node_types_dense(adj_mat) + + return adj_mat + + +def norm_adj_mat_one_node_type(adj_mat: torch.Tensor) -> torch.Tensor: + _check_tensor(adj_mat) + _check_square(adj_mat) + + if adj_mat.is_sparse: + return norm_adj_mat_one_node_type_sparse(adj_mat) + else: + return norm_adj_mat_one_node_type_dense(adj_mat) + + +def norm_adj_mat_two_node_types_sparse(adj_mat: torch.Tensor) -> torch.Tensor: + _check_tensor(adj_mat) + _check_sparse(adj_mat) + _check_2d(adj_mat) + + adj_mat = adj_mat.coalesce() + indices = adj_mat.indices() + values = adj_mat.values() + degrees_row = torch.zeros(adj_mat.shape[0], device=adj_mat.device) + degrees_row = degrees_row.index_add(0, indices[0], values.to(degrees_row.dtype)) + degrees_col = torch.zeros(adj_mat.shape[1], device=adj_mat.device) + degrees_col = degrees_col.index_add(0, indices[1], values.to(degrees_col.dtype)) + values = values.to(degrees_row.dtype) / torch.sqrt(degrees_row[indices[0]] * degrees_col[indices[1]]) + adj_mat = _sparse_coo_tensor(indices, values, adj_mat.shape) + + return adj_mat + + +def norm_adj_mat_two_node_types_dense(adj_mat: torch.Tensor) -> torch.Tensor: + _check_tensor(adj_mat) + _check_dense(adj_mat) + _check_2d(adj_mat) + + degrees_row = adj_mat.sum(1).view(-1, 1).to(torch.float32) + degrees_col = adj_mat.sum(0).view(1, -1).to(torch.float32) + degrees_row = torch.sqrt(degrees_row) + degrees_col = torch.sqrt(degrees_col) + adj_mat = adj_mat.to(degrees_row.dtype) / degrees_row + adj_mat = adj_mat / degrees_col + + return adj_mat + + +def norm_adj_mat_two_node_types(adj_mat: torch.Tensor) -> torch.Tensor: + _check_tensor(adj_mat) + _check_2d(adj_mat) + + if adj_mat.is_sparse: + return norm_adj_mat_two_node_types_sparse(adj_mat) + else: + return norm_adj_mat_two_node_types_dense(adj_mat) diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py new file mode 100644 index 0000000..7c55944 --- /dev/null +++ b/src/triacontagon/sampling.py @@ -0,0 +1,47 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import numpy as np +import torch +import torch.utils.data +from typing import List, \ + Union + + +def fixed_unigram_candidate_sampler( + true_classes: Union[np.array, torch.Tensor], + unigrams: List[Union[int, float]], + distortion: float = 1.): + + if isinstance(true_classes, torch.Tensor): + true_classes = true_classes.detach().cpu().numpy() + + if isinstance(unigrams, torch.Tensor): + unigrams = unigrams.detach().cpu().numpy() + + if len(true_classes.shape) != 2: + raise ValueError('true_classes must be a 2D matrix with shape (num_samples, num_true)') + + num_samples = true_classes.shape[0] + unigrams = np.array(unigrams) + if distortion != 1.: + unigrams = unigrams.astype(np.float64) ** distortion + # print('unigrams:', unigrams) + indices = np.arange(num_samples) + result = np.zeros(num_samples, dtype=np.int64) + while len(indices) > 0: + # print('len(indices):', len(indices)) + sampler = torch.utils.data.WeightedRandomSampler(unigrams, len(indices)) + candidates = np.array(list(sampler)) + candidates = np.reshape(candidates, (len(indices), 1)) + # print('candidates:', candidates) + # print('true_classes:', true_classes[indices, :]) + result[indices] = candidates.T + mask = (candidates == true_classes[indices, :]) + mask = mask.sum(1).astype(np.bool) + # print('mask:', mask) + indices = indices[mask] + return torch.tensor(result) diff --git a/src/triacontagon/trainprep.py b/src/triacontagon/trainprep.py new file mode 100644 index 0000000..c49300a --- /dev/null +++ b/src/triacontagon/trainprep.py @@ -0,0 +1,215 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +from .sampling import fixed_unigram_candidate_sampler +import torch +from dataclasses import dataclass, \ + field +from typing import Any, \ + List, \ + Tuple, \ + Dict +from .data import NodeType, \ + RelationType, \ + RelationTypeBase, \ + RelationFamily, \ + RelationFamilyBase, \ + Data +from collections import defaultdict +from .normalize import norm_adj_mat_one_node_type, \ + norm_adj_mat_two_node_types +import numpy as np + + +@dataclass +class TrainValTest(object): + train: Any + val: Any + test: Any + + +@dataclass +class PreparedRelationType(RelationTypeBase): + edges_pos: TrainValTest + edges_neg: TrainValTest + edges_back_pos: TrainValTest + edges_back_neg: TrainValTest + + +@dataclass +class PreparedRelationFamily(RelationFamilyBase): + relation_types: List[PreparedRelationType] + + +@dataclass +class PreparedData(object): + node_types: List[NodeType] + relation_families: List[PreparedRelationFamily] + + +def _empty_edge_list_tvt() -> TrainValTest: + return TrainValTest(*[ torch.zeros((0, 2), dtype=torch.long) for _ in range(3) ]) + + +def train_val_test_split_edges(edges: torch.Tensor, + ratios: TrainValTest) -> TrainValTest: + + if not isinstance(edges, torch.Tensor): + raise ValueError('edges must be a torch.Tensor') + + if len(edges.shape) != 2 or edges.shape[1] != 2: + raise ValueError('edges shape must be (num_edges, 2)') + + if not isinstance(ratios, TrainValTest): + raise ValueError('ratios must be a TrainValTest') + + if ratios.train + ratios.val + ratios.test != 1.0: + raise ValueError('Train, validation and test ratios must add up to 1') + + order = torch.randperm(len(edges)) + edges = edges[order, :] + n = round(len(edges) * ratios.train) + edges_train = edges[:n] + n_1 = round(len(edges) * (ratios.train + ratios.val)) + edges_val = edges[n:n_1] + edges_test = edges[n_1:] + + return TrainValTest(edges_train, edges_val, edges_test) + + +def get_edges_and_degrees(adj_mat: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + if adj_mat.is_sparse: + adj_mat = adj_mat.coalesce() + degrees = torch.zeros(adj_mat.shape[1], dtype=torch.int64, + device=adj_mat.device) + degrees = degrees.index_add(0, adj_mat.indices()[1], + torch.ones(adj_mat.indices().shape[1], dtype=torch.int64, + device=adj_mat.device)) + edges_pos = adj_mat.indices().transpose(0, 1) + else: + degrees = adj_mat.sum(0) + edges_pos = torch.nonzero(adj_mat) + return edges_pos, degrees + + +def prepare_adj_mat(adj_mat: torch.Tensor, + ratios: TrainValTest) -> Tuple[TrainValTest, TrainValTest]: + + if not isinstance(adj_mat, torch.Tensor): + raise ValueError('adj_mat must be a torch.Tensor') + + edges_pos, degrees = get_edges_and_degrees(adj_mat) + + neg_neighbors = fixed_unigram_candidate_sampler( + edges_pos[:, 1].view(-1, 1), degrees, 0.75).to(adj_mat.device) + print(edges_pos.dtype) + print(neg_neighbors.dtype) + edges_neg = torch.cat((edges_pos[:, 0].view(-1, 1), neg_neighbors.view(-1, 1)), 1) + + edges_pos = train_val_test_split_edges(edges_pos, ratios) + edges_neg = train_val_test_split_edges(edges_neg, ratios) + + adj_mat_train = torch.sparse_coo_tensor(indices = edges_pos.train.transpose(0, 1), + values=torch.ones(len(edges_pos.train)), size=adj_mat.shape, dtype=adj_mat.dtype, + device=adj_mat.device) + + return adj_mat_train, edges_pos, edges_neg + + +def prep_rel_one_node_type(r: RelationType, + ratios: TrainValTest) -> PreparedRelationType: + + adj_mat = r.adjacency_matrix + adj_mat_train, edges_pos, edges_neg = prepare_adj_mat(adj_mat, ratios) + adj_mat_back_train, edges_back_pos, edges_back_neg = \ + None, _empty_edge_list_tvt(), _empty_edge_list_tvt() + + print('adj_mat_train:', adj_mat_train) + adj_mat_train = norm_adj_mat_one_node_type(adj_mat_train) + + return PreparedRelationType(r.name, r.node_type_row, r.node_type_column, + adj_mat_train, adj_mat_back_train, edges_pos, edges_neg, + edges_back_pos, edges_back_neg) + + +def prep_rel_two_node_types_sym(r: RelationType, + ratios: TrainValTest) -> PreparedRelationType: + + adj_mat = r.adjacency_matrix + adj_mat_train, edges_pos, edges_neg = prepare_adj_mat(adj_mat, ratios) + edges_back_pos, edges_back_neg = \ + _empty_edge_list_tvt(), _empty_edge_list_tvt() + + return PreparedRelationType(r.name, r.node_type_row, + r.node_type_column, + norm_adj_mat_two_node_types(adj_mat_train), + norm_adj_mat_two_node_types(adj_mat_train.transpose(0, 1)), + edges_pos, edges_neg, edges_back_pos, edges_back_neg) + + +def prep_rel_two_node_types_asym(r: RelationType, + ratios: TrainValTest) -> PreparedRelationType: + + if r.adjacency_matrix is not None: + adj_mat_train, edges_pos, edges_neg =\ + prepare_adj_mat(r.adjacency_matrix, ratios) + else: + adj_mat_train, edges_pos, edges_neg = \ + None, _empty_edge_list_tvt(), _empty_edge_list_tvt() + + if r.adjacency_matrix_backward is not None: + adj_mat_back_train, edges_back_pos, edges_back_neg = \ + prepare_adj_mat(r.adjacency_matrix_backward, ratios) + else: + adj_mat_back_train, edges_back_pos, edges_back_neg = \ + None, _empty_edge_list_tvt(), _empty_edge_list_tvt() + + return PreparedRelationType(r.name, r.node_type_row, + r.node_type_column, + norm_adj_mat_two_node_types(adj_mat_train), + norm_adj_mat_two_node_types(adj_mat_back_train), + edges_pos, edges_neg, edges_back_pos, edges_back_neg) + + +def prepare_relation_type(r: RelationType, + ratios: TrainValTest, is_symmetric: bool) -> PreparedRelationType: + + if not isinstance(r, RelationType): + raise ValueError('r must be a RelationType') + + if not isinstance(ratios, TrainValTest): + raise ValueError('ratios must be a TrainValTest') + + if r.node_type_row == r.node_type_column: + return prep_rel_one_node_type(r, ratios) + elif is_symmetric: + return prep_rel_two_node_types_sym(r, ratios) + else: + return prep_rel_two_node_types_asym(r, ratios) + + +def prepare_relation_family(fam: RelationFamily, + ratios: TrainValTest) -> PreparedRelationFamily: + + relation_types = [] + + for r in fam.relation_types: + relation_types.append(prepare_relation_type(r, ratios, fam.is_symmetric)) + + return PreparedRelationFamily(fam.data, fam.name, + fam.node_type_row, fam.node_type_column, + fam.is_symmetric, fam.decoder_class, + relation_types) + + +def prepare_training(data: Data, ratios: TrainValTest) -> PreparedData: + if not isinstance(data, Data): + raise ValueError('data must be of class Data') + + relation_families = [ prepare_relation_family(fam, ratios) \ + for fam in data.relation_families ] + + return PreparedData(data.node_types, relation_families) diff --git a/src/triacontagon/weights.py b/src/triacontagon/weights.py new file mode 100644 index 0000000..2dcb7b4 --- /dev/null +++ b/src/triacontagon/weights.py @@ -0,0 +1,19 @@ +# +# Copyright (C) Stanislaw Adaszewski, 2020 +# License: GPLv3 +# + + +import torch +import numpy as np + + +def init_glorot(in_channels, out_channels, dtype=torch.float32): + """Create a weight variable with Glorot & Bengio (AISTATS 2010) + initialization. + """ + init_range = np.sqrt(6.0 / (in_channels + out_channels)) + initial = -init_range + 2 * init_range * \ + torch.rand(( in_channels, out_channels ), dtype=dtype) + initial = initial.requires_grad_(True) + return initial -- 2.26.2 From 4f953fd203a19e17743bb9ed41eace9ea7900052 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 4 Aug 2020 14:11:59 +0200 Subject: [PATCH 177/227] Work on triacontagon. --- src/triacontagon/__init__.py | 0 src/triacontagon/data.py | 218 ++++++-------------------------- src/triacontagon/decode.py | 134 +++++--------------- src/triacontagon/model.py | 129 +++++++++++++++++++ src/triacontagon/util.py | 174 +++++++++++++++++++++++++ tests/triacontagon/test_util.py | 95 ++++++++++++++ 6 files changed, 470 insertions(+), 280 deletions(-) create mode 100644 src/triacontagon/__init__.py create mode 100644 src/triacontagon/model.py create mode 100644 src/triacontagon/util.py create mode 100644 tests/triacontagon/test_util.py diff --git a/src/triacontagon/__init__.py b/src/triacontagon/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/triacontagon/data.py b/src/triacontagon/data.py index 4505adf..22a4c89 100644 --- a/src/triacontagon/data.py +++ b/src/triacontagon/data.py @@ -4,206 +4,68 @@ # -from collections import defaultdict -from dataclasses import dataclass, field -import torch -from typing import List, \ - Dict, \ +from dataclasses import dataclass +from typing import Callable, \ Tuple, \ - Any, \ - Type -from .decode import DEDICOMDecoder, \ - BilinearDecoder -import numpy as np - - -def _equal(x: torch.Tensor, y: torch.Tensor): - if x.is_sparse ^ y.is_sparse: - raise ValueError('Cannot mix sparse and dense tensors') - - if not x.is_sparse: - return (x == y) - - return ((x - y).coalesce().values() == 0) + List +import types +from .util import _nonzero_sum @dataclass -class NodeType(object): - name: str - count: int +class DecodingMatrices(object): + global_interaction: torch.Tensor + local_variation: torch.Tensor @dataclass -class RelationTypeBase(object): +class VertexType(object): name: str - node_type_row: int - node_type_column: int - adjacency_matrix: torch.Tensor - adjacency_matrix_backward: torch.Tensor - - -@dataclass -class RelationType(RelationTypeBase): - pass + count: int @dataclass -class RelationFamilyBase(object): - data: 'Data' +class EdgeType(object): name: str - node_type_row: int - node_type_column: int - is_symmetric: bool - decoder_class: Type - - -@dataclass -class RelationFamily(RelationFamilyBase): - relation_types: List[RelationType] = None - - def __post_init__(self) -> None: - if not self.is_symmetric and \ - self.decoder_class != DEDICOMDecoder and \ - self.decoder_class != BilinearDecoder: - raise TypeError('Family is assymetric but the specified decoder_class supports symmetric relations only') - - self.relation_types = [] - - def add_relation_type(self, - name: str, adjacency_matrix: torch.Tensor, - adjacency_matrix_backward: torch.Tensor = None) -> None: - - name = str(name) - node_type_row = self.node_type_row - node_type_column = self.node_type_column - - if adjacency_matrix is None and adjacency_matrix_backward is None: - raise ValueError('adjacency_matrix and adjacency_matrix_backward cannot both be None') - - if adjacency_matrix is not None and \ - not isinstance(adjacency_matrix, torch.Tensor): - raise ValueError('adjacency_matrix must be a torch.Tensor') - - if adjacency_matrix_backward is not None \ - and not isinstance(adjacency_matrix_backward, torch.Tensor): - raise ValueError('adjacency_matrix_backward must be a torch.Tensor') - - if adjacency_matrix is not None and \ - adjacency_matrix.shape != (self.data.node_types[node_type_row].count, - self.data.node_types[node_type_column].count): - raise ValueError('adjacency_matrix shape must be (num_row_nodes, num_column_nodes)') - - if adjacency_matrix_backward is not None and \ - adjacency_matrix_backward.shape != (self.data.node_types[node_type_column].count, - self.data.node_types[node_type_row].count): - raise ValueError('adjacency_matrix_backward shape must be (num_column_nodes, num_row_nodes)') - - if node_type_row == node_type_column and \ - adjacency_matrix_backward is not None: - raise ValueError('Relation between nodes of the same type must be expressed using a single matrix') - - if self.is_symmetric and adjacency_matrix_backward is not None: - raise ValueError('Cannot use a custom adjacency_matrix_backward in a symmetric relation family') - - if self.is_symmetric and node_type_row == node_type_column and \ - not torch.all(_equal(adjacency_matrix, - adjacency_matrix.transpose(0, 1))): - raise ValueError('Relation family is symmetric but adjacency_matrix is assymetric') - - if not self.is_symmetric and node_type_row != node_type_column and \ - adjacency_matrix_backward is None: - raise ValueError('Relation is asymmetric but adjacency_matrix_backward is None') - - if self.is_symmetric and node_type_row != node_type_column: - adjacency_matrix_backward = adjacency_matrix.transpose(0, 1) - - self.relation_types.append(RelationType(name, - node_type_row, node_type_column, - adjacency_matrix, adjacency_matrix_backward)) - - def node_name(self, index): - return self.data.node_types[index].name - - def __repr__(self): - s = 'Relation family %s' % self.name - - for r in self.relation_types: - s += '\n - %s%s' % (r.name, ' (two-way)' \ - if (r.adjacency_matrix is not None \ - and r.adjacency_matrix_backward is not None) \ - or self.node_type_row == self.node_type_column \ - else '%s <- %s' % (self.node_name(self.node_type_row), - self.node_name(self.node_type_column))) - - return s - - def repr_indented(self): - s = ' - %s' % self.name - - for r in self.relation_types: - s += '\n - %s%s' % (r.name, ' (two-way)' \ - if (r.adjacency_matrix is not None \ - and r.adjacency_matrix_backward is not None) \ - or self.node_type_row == self.node_type_column \ - else '%s <- %s' % (self.node_name(self.node_type_row), - self.node_name(self.node_type_column))) - - return s + vertex_type_row: int + vertex_type_column: int + adjacency_matrices: List[torch.Tensor] + decoder_factory: Callable[[], DecodingMatrices] + total_connectivity: torch.Tensor class Data(object): - node_types: List[NodeType] - relation_families: List[RelationFamily] + vertex_types: List[VertexType] + edge_types: List[EdgeType] def __init__(self) -> None: - self.node_types = [] - self.relation_families = [] + self.vertex_types = [] + self.edge_types = {} - def add_node_type(self, name: str, count: int) -> None: + def add_vertex_type(self, name: str, count: int) -> None: name = str(name) count = int(count) if not name: - raise ValueError('You must provide a non-empty node type name') + raise ValueError('You must provide a non-empty vertex type name') if count <= 0: - raise ValueError('You must provide a positive node count') - self.node_types.append(NodeType(name, count)) + raise ValueError('You must provide a positive vertex count') + self.vertex_types.append(VertexType(name, count)) - def add_relation_family(self, name: str, node_type_row: int, - node_type_column: int, is_symmetric: bool, - decoder_class: Type = DEDICOMDecoder): + def add_edge_type(self, name: str, + vertex_type_row: int, vertex_type_column: int, + adjacency_matrices: List[torch.Tensor], + decoder_factory: Callable[[], DecodingMatrices]) -> None: name = str(name) - node_type_row = int(node_type_row) - node_type_column = int(node_type_column) - is_symmetric = bool(is_symmetric) - - if node_type_row < 0 or node_type_row >= len(self.node_types): - raise ValueError('node_type_row outside of the valid range of node types') - - if node_type_column < 0 or node_type_column >= len(self.node_types): - raise ValueError('node_type_column outside of the valid range of node types') - - fam = RelationFamily(self, name, node_type_row, node_type_column, - is_symmetric, decoder_class) - self.relation_families.append(fam) - - return fam - - def __repr__(self): - n = len(self.node_types) - if n == 0: - return 'Empty Icosagon Data' - s = '' - s += 'Icosagon Data with:\n' - s += '- ' + str(n) + ' node type(s):\n' - for nt in self.node_types: - s += ' - ' + nt.name + '\n' - if len(self.relation_families) == 0: - s += '- No relation families\n' - return s.strip() - - s += '- %d relation families:\n' % len(self.relation_families) - for fam in self.relation_families: - s += fam.repr_indented() + '\n' - - return s.strip() + vertex_type_row = int(vertex_type_row) + vertex_type_column = int(vertex_type_column) + if not isinstance(adjacency_matrices, list): + raise TypeError('adjacency_matrices must be a list of tensors') + if not isinstance(decoder_factory, types.FunctionType): + raise TypeError('decoder_factory must be a function') + if (vertex_type_row, vertex_type_column) in self.edge_types: + raise KeyError('Edge type for given combination of row and column already exists') + total_connectivity = _nonzero_sum(adjacency_matrices) + self.edges_types[vertex_type_row, vertex_type_column] = \ + VertexType(name, vertex_type_row, vertex_type_column, + adjacency_matrices, decoder_factory, total_connectivity) diff --git a/src/triacontagon/decode.py b/src/triacontagon/decode.py index 00df8b2..25ae822 100644 --- a/src/triacontagon/decode.py +++ b/src/triacontagon/decode.py @@ -7,117 +7,47 @@ import torch from .weights import init_glorot from .dropout import dropout +from typing import Tuple, \ + List -class DEDICOMDecoder(torch.nn.Module): - """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" - def __init__(self, input_dim, num_relation_types, keep_prob=1., - activation=torch.sigmoid, **kwargs): +def dedicom_decoder(input_dim: int, num_relation_types: int) -> + Tuple[torch.Tensor, List[torch.Tensor]]: - super().__init__(**kwargs) - self.input_dim = input_dim - self.num_relation_types = num_relation_types - self.keep_prob = keep_prob - self.activation = activation + global_interaction = init_glorot(input_dim, input_dim) + local_variation = [ + torch.diag(torch.flatten(init_glorot(input_dim, 1))) \ + for _ in range(num_relation_types) + ] + return (global_interaction, local_variation) - self.global_interaction = torch.nn.Parameter(init_glorot(input_dim, input_dim)) - self.local_variation = torch.nn.ParameterList([ - torch.nn.Parameter(torch.flatten(init_glorot(input_dim, 1))) \ - for _ in range(num_relation_types) - ]) - def forward(self, inputs_row, inputs_col, relation_index): - inputs_row = dropout(inputs_row, self.keep_prob) - inputs_col = dropout(inputs_col, self.keep_prob) +def dist_mult_decoder(input_dim: int, num_relation_types: int) -> + Tuple[torch.Tensor, List[torch.Tensor]]: - relation = torch.diag(self.local_variation[relation_index]) + global_interaction = torch.eye(input_dim, input_dim) + local_variation = [ + torch.diag(torch.flatten(init_glorot(input_dim, 1)))) \ + for _ in range(num_relation_types) + ] + return (global_interaction, local_variation) - product1 = torch.mm(inputs_row, relation) - product2 = torch.mm(product1, self.global_interaction) - product3 = torch.mm(product2, relation) - rec = torch.bmm(product3.view(product3.shape[0], 1, product3.shape[1]), - inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) - rec = torch.flatten(rec) - return self.activation(rec) +def bilinear_decoder(input_dim: int, num_relation_types: int) -> + Tuple[torch.Tensor, List[torch.Tensor]]: + global_interaction = torch.eye(input_dim, input_dim) + local_variation = [ + init_glorot(input_dim, input_dim) \ + for _ in range(num_relation_types) + ] + return (global_interaction, local_variation) -class DistMultDecoder(torch.nn.Module): - """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" - def __init__(self, input_dim, num_relation_types, keep_prob=1., - activation=torch.sigmoid, **kwargs): - super().__init__(**kwargs) - self.input_dim = input_dim - self.num_relation_types = num_relation_types - self.keep_prob = keep_prob - self.activation = activation +def inner_product_decoder(input_dim: int, num_relation_types: int) -> + Tuple[torch.Tensor, List[torch.Tensor]]: - self.relation = torch.nn.ParameterList([ - torch.nn.Parameter(torch.flatten(init_glorot(input_dim, 1))) \ - for _ in range(num_relation_types) - ]) - - def forward(self, inputs_row, inputs_col, relation_index): - inputs_row = dropout(inputs_row, self.keep_prob) - inputs_col = dropout(inputs_col, self.keep_prob) - - relation = torch.diag(self.relation[relation_index]) - - intermediate_product = torch.mm(inputs_row, relation) - rec = torch.bmm(intermediate_product.view(intermediate_product.shape[0], 1, intermediate_product.shape[1]), - inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) - rec = torch.flatten(rec) - - return self.activation(rec) - - -class BilinearDecoder(torch.nn.Module): - """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" - def __init__(self, input_dim, num_relation_types, keep_prob=1., - activation=torch.sigmoid, **kwargs): - - super().__init__(**kwargs) - self.input_dim = input_dim - self.num_relation_types = num_relation_types - self.keep_prob = keep_prob - self.activation = activation - - self.relation = torch.nn.ParameterList([ - torch.nn.Parameter(init_glorot(input_dim, input_dim)) \ - for _ in range(num_relation_types) - ]) - - def forward(self, inputs_row, inputs_col, relation_index): - inputs_row = dropout(inputs_row, self.keep_prob) - inputs_col = dropout(inputs_col, self.keep_prob) - - intermediate_product = torch.mm(inputs_row, self.relation[relation_index]) - rec = torch.bmm(intermediate_product.view(intermediate_product.shape[0], 1, intermediate_product.shape[1]), - inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) - rec = torch.flatten(rec) - - return self.activation(rec) - - -class InnerProductDecoder(torch.nn.Module): - """DEDICOM Tensor Factorization Decoder model layer for link prediction.""" - def __init__(self, input_dim, num_relation_types, keep_prob=1., - activation=torch.sigmoid, **kwargs): - - super().__init__(**kwargs) - self.input_dim = input_dim - self.num_relation_types = num_relation_types - self.keep_prob = keep_prob - self.activation = activation - - - def forward(self, inputs_row, inputs_col, _): - inputs_row = dropout(inputs_row, self.keep_prob) - inputs_col = dropout(inputs_col, self.keep_prob) - - rec = torch.bmm(inputs_row.view(inputs_row.shape[0], 1, inputs_row.shape[1]), - inputs_col.view(inputs_col.shape[0], inputs_col.shape[1], 1)) - rec = torch.flatten(rec) - - return self.activation(rec) + global_interaction = torch.eye(input_dim, input_dim) + local_variation = torch.eye(input_dim, input_dim) + local_variation = [ local_variation ] * num_relation_types + return (global_interaction, local_variation) diff --git a/src/triacontagon/model.py b/src/triacontagon/model.py new file mode 100644 index 0000000..1b98931 --- /dev/null +++ b/src/triacontagon/model.py @@ -0,0 +1,129 @@ +from .data import Data, \ + EdgeType +import torch +from dataclasses import dataclass +from .weights import init_glorot +import types +from typing import List, \ + Dict, \ + Callable +from .util import _sparse_coo_tensor + + +@dataclass +class TrainingBatch(object): + vertex_type_row: int + vertex_type_column: int + relation_type_index: int + edges: torch.Tensor + + +class Model(torch.nn.Module): + def __init__(self, data: Data, layer_dimensions: List[int], + keep_prob: float, + conv_activation: Callable[[torch.Tensor], torch.Tensor], + dec_activation: Callable[[torch.Tensor], torch.Tensor], + **kwargs) -> None: + super().__init__(**kwargs) + + if not isinstance(data, Data): + raise TypeError('data must be an instance of Data') + + if not isinstance(conv_activation, types.FunctionType): + raise TypeError('conv_activation must be a function') + + if not isinstance(dec_activation, types.FunctionType): + raise TypeError('dec_activation must be a function') + + self.data = data + self.layer_dimensions = list(layer_dimensions) + self.keep_prob = float(keep_prob) + self.conv_activation = conv_activation + self.dec_activation = dec_activation + + self.conv_weights = None + self.dec_weights = None + self.build() + + + def build(self) -> None: + self.conv_weights = torch.nn.ParameterDict() + for i in range(len(self.layer_dimensions) - 1): + in_dimension = self.layer_dimensions[i] + out_dimension = self.layer_dimensions[i + 1] + + for _, et in self.data.edge_types.items(): + weight = init_glorot(in_dimension, out_dimension) + self.conv_weights[et.vertex_type_row, et.vertex_type_column, i] = \ + torch.nn.Parameter(weight) + + self.dec_weights = torch.nn.ParameterDict() + for _, et in self.data.edge_types.items(): + global_interaction, local_variation = \ + et.decoder_factory(self.layer_dimensions[-1], + len(et.adjacency_matrices)) + self.dec_weights[et.vertex_type_row, et.vertex_type_column] = \ + torch.nn.ParameterList([ + torch.nn.Parameter(global_interaction), + torch.nn.Parameter(local_variation) + ]) + + def limit_adjacency_matrix_to_rows(self, adjacency_matrix: torch.Tensor, + rows: torch.Tensor) -> torch.Tensor: + + adj_mat = adjacency_matrix.coalesce() + adj_mat = torch.index_select(adj_mat, 0, rows) + + adj_mat = adj_mat.coalesce() + indices = adj_mat.indices() + indices[0] = rows + + adj_mat = _sparse_coo_tensor(indices, adj_mat.values(), adjacency_matrix.shape) + + def temporary_adjacency_matrix(self, adjacency_matrix: torch.Tensor, + batch: TrainingBatch, total_connectivity: torch.Tensor) -> torch.Tensor: + + col = batch.vertex_type_column + rows = batch.edges[:, 0] + + columns = batch.edges[:, 1].sum(dim=0).flatten() + columns = torch.nonzero(columns) + + for i in range(len(self.layer_dimensions) - 1): + + columns = + + + def temporary_adjacency_matrices(self, batch: TrainingBatch) -> + Dict[Tuple[int, int], List[List[torch.Tensor]]]: + + col = batch.vertex_type_column + batch.edges[:, 1] + + res = {} + + for _, et in self.data.edge_types.items(): + sum_nonzero = _nonzero_sum(et.adjacency_matrices) + res[et.vertex_type_row, et.vertex_type_column] = \ + [ self.temporary_adjacency_matrix(adj_mat, batch, + et.total_connectivity) \ + for adj_mat in et.adjacency_matrices ] + + return res + + def forward(self, initial_repr: List[torch.Tensor], + batch: TrainingBatch) -> torch.Tensor: + + if not isinstance(initial_repr, list): + raise TypeError('initial_repr must be a list') + + if len(initial_repr) != len(self.data.vertex_types): + raise ValueError('initial_repr must contain representations for all vertex types') + + if not isinstance(batch, TrainingBatch): + raise TypeError('batch must be an instance of TrainingBatch') + + adj_matrices = self.temporary_adjacency_matrices(batch) + + row_vertices = initial_repr[batch.vertex_type_row] + column_vertices = initial_repr[batch.vertex_type_column] diff --git a/src/triacontagon/util.py b/src/triacontagon/util.py new file mode 100644 index 0000000..2367b06 --- /dev/null +++ b/src/triacontagon/util.py @@ -0,0 +1,174 @@ +import torch +from typing import List, \ + Set +import time + + +def _equal(x: torch.Tensor, y: torch.Tensor): + if x.is_sparse ^ y.is_sparse: + raise ValueError('Cannot mix sparse and dense tensors') + + if not x.is_sparse: + return (x == y) + + return ((x - y).coalesce().values() == 0) + + +def _sparse_coo_tensor(indices, values, size): + ctor = { torch.float32: torch.sparse.FloatTensor, + torch.float32: torch.sparse.DoubleTensor, + torch.uint8: torch.sparse.ByteTensor, + torch.long: torch.sparse.LongTensor, + torch.int: torch.sparse.IntTensor, + torch.short: torch.sparse.ShortTensor, + torch.bool: torch.sparse.ByteTensor }[values.dtype] + return ctor(indices, values, size) + + +def _nonzero_sum(adjacency_matrices: List[torch.Tensor]): + if len(adjacency_matrices) == 0: + raise ValueError('adjacency_matrices must be non-empty') + + if not all([x.is_sparse for x in adjacency_matrices]): + raise ValueError('All adjacency matrices must be sparse') + + indices = [ x.indices() for x in adjacency_matrices ] + indices = torch.cat(indices, dim=1) + + values = torch.ones(indices.shape[1]) + res = _sparse_coo_tensor(indices, values, adjacency_matrices[0].shape) + res = res.coalesce() + + indices = res.indices() + res = _sparse_coo_tensor(indices, + torch.ones(indices.shape[1], dtype=torch.uint8)) + + return res + + +def _clear_adjacency_matrix_except_rows(adjacency_matrix: torch.Tensor, + rows: torch.Tensor, row_vertex_count: int, num_relation_types: int) -> torch.Tensor: + + if not adjacency_matrix.is_sparse: + raise ValueError('adjacency_matrix must be sparse') + + if not adjacency_matrix.shape[0] == row_vertex_count * num_relation_types: + raise ValueError('adjacency_matrix must have as many rows as row vertex count times number of relation types') + + t = time.time() + rows = [ rows + row_vertex_count * i \ + for i in range(num_relation_types) ] + print('rows took:', time.time() - t) + t = time.time() + rows = torch.cat(rows) + print('cat took:', time.time() - t) + # print('rows:', rows) + rows = set(rows.tolist()) + # print('rows:', rows) + + t = time.time() + adj_mat = adjacency_matrix.coalesce() + indices = adj_mat.indices() + values = adj_mat.values() + print('indices[0]:', indices[0]) + print('indices[0][1]:', indices[0][1], indices[0][1] in rows) + selection = torch.tensor([ (idx.item() in rows) for idx in indices[0] ]) + # print('selection:', selection) + selection = torch.nonzero(selection, as_tuple=True)[0] + # print('selection:', selection) + indices = indices[:, selection] + values = values[selection] + print('"index_select()" took:', time.time() - t) + + t = time.time() + res = _sparse_coo_tensor(indices, values, adjacency_matrix.shape) + print('_sparse_coo_tensor() took:', time.time() - t) + + return res + + # t = time.time() + # adj_mat = torch.index_select(adjacency_matrix, 0, rows) + # print('index_select took:', time.time() - t) + + t = time.time() + adj_mat = adj_mat.coalesce() + print('coalesce() took:', time.time() - t) + indices = adj_mat.indices() + + # print('indices:', indices) + + values = adj_mat.values() + t = time.time() + indices[0] = rows[indices[0]] + print('Lookup took:', time.time() - t) + + t = time.time() + adj_mat = _sparse_coo_tensor(indices, values, adjacency_matrix.shape) + print('_sparse_coo_tensor() took:', time.time() - t) + + return adj_mat + + +def _sparse_diag_cat(matrices: List[torch.Tensor]): + if len(matrices) == 0: + raise ValueError('The list of matrices must be non-empty') + + if not all(m.is_sparse for m in matrices): + raise ValueError('All matrices must be sparse') + + if not all(len(m.shape) == 2 for m in matrices): + raise ValueError('All matrices must be 2D') + + indices = [] + values = [] + row_offset = 0 + col_offset = 0 + + for m in matrices: + ind = m._indices().clone() + ind[0] += row_offset + ind[1] += col_offset + indices.append(ind) + values.append(m._values()) + row_offset += m.shape[0] + col_offset += m.shape[1] + + indices = torch.cat(indices, dim=1) + values = torch.cat(values) + + return _sparse_coo_tensor(indices, values, size=(row_offset, col_offset)) + + +def _cat(matrices: List[torch.Tensor]): + if len(matrices) == 0: + raise ValueError('Empty list passed to _cat()') + + n = sum(a.is_sparse for a in matrices) + if n != 0 and n != len(matrices): + raise ValueError('All matrices must have the same layout (dense or sparse)') + + if not all(a.shape[1:] == matrices[0].shape[1:] for a in matrices): + raise ValueError('All matrices must have the same dimensions apart from dimension 0') + + if not matrices[0].is_sparse: + return torch.cat(matrices) + + total_rows = sum(a.shape[0] for a in matrices) + indices = [] + values = [] + row_offset = 0 + + for a in matrices: + ind = a._indices().clone() + val = a._values() + ind[0] += row_offset + ind = ind.transpose(0, 1) + indices.append(ind) + values.append(val) + row_offset += a.shape[0] + + indices = torch.cat(indices).transpose(0, 1) + values = torch.cat(values) + + res = _sparse_coo_tensor(indices, values, size=(row_offset, matrices[0].shape[1])) + return res diff --git a/tests/triacontagon/test_util.py b/tests/triacontagon/test_util.py new file mode 100644 index 0000000..5937535 --- /dev/null +++ b/tests/triacontagon/test_util.py @@ -0,0 +1,95 @@ +from triacontagon.util import \ + _clear_adjacency_matrix_except_rows, \ + _sparse_diag_cat, \ + _equal +import torch +import time + + +def test_clear_adjacency_matrix_except_rows_01(): + adj_mat = torch.tensor([ + [0, 0, 1, 0, 0], + [0, 0, 0, 1, 1], + [1, 0, 1, 0, 0], + [1, 1, 0, 0, 0] + ], dtype=torch.uint8).to_sparse() + + adj_mat = _sparse_diag_cat([ adj_mat, adj_mat ]) + + res = _clear_adjacency_matrix_except_rows(adj_mat, + torch.tensor([1, 3]), 4, 2) + + res = res.to_dense() + + truth = torch.tensor([ + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 1, 1, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 1, 1], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 1, 1, 0, 0, 0] + ], dtype=torch.uint8) + + print('res:', res) + + assert torch.all(res == truth) + + +def test_clear_adjacency_matrix_except_rows_02(): + adj_mat = torch.rand(6, 10).round().to(torch.uint8) + + t = time.time() + res = _sparse_diag_cat([ adj_mat.to_sparse() ] * 130) + print('_sparse_diag_cat() took:', time.time() - t) + + t = time.time() + res = _clear_adjacency_matrix_except_rows(res, torch.tensor([1, 3, 5]), + 6, 130) + print('_clear_adjacency_matrix_except_rows() took:', time.time() - t) + + adj_mat[0] = adj_mat[2] = adj_mat[4] = \ + torch.zeros(10) + truth = _sparse_diag_cat([ adj_mat.to_sparse() ] * 130) + + assert _equal(res, truth).all() + + +def test_clear_adjacency_matrix_except_rows_03(): + adj_mat = torch.rand(6, 10).round().to(torch.uint8) + + t = time.time() + res = _sparse_diag_cat([ adj_mat.to_sparse() ] * 1300) + print('_sparse_diag_cat() took:', time.time() - t) + + t = time.time() + res = _clear_adjacency_matrix_except_rows(res, torch.tensor([1, 3, 5]), + 6, 1300) + print('_clear_adjacency_matrix_except_rows() took:', time.time() - t) + + adj_mat[0] = adj_mat[2] = adj_mat[4] = \ + torch.zeros(10) + truth = _sparse_diag_cat([ adj_mat.to_sparse() ] * 1300) + + assert _equal(res, truth).all() + + +def test_clear_adjacency_matrix_except_rows_04(): + adj_mat = (torch.rand(2000, 2000) < 0.001).to(torch.uint8) + + t = time.time() + res = _sparse_diag_cat([ adj_mat.to_sparse() ] * 1300) + print('_sparse_diag_cat() took:', time.time() - t) + + t = time.time() + res = _clear_adjacency_matrix_except_rows(res, torch.tensor([1, 3, 5]), + 2000, 1300) + print('_clear_adjacency_matrix_except_rows() took:', time.time() - t) + + adj_mat[0] = adj_mat[2] = adj_mat[4] = \ + torch.zeros(2000) + adj_mat[6:] = torch.zeros(2000) + truth = _sparse_diag_cat([ adj_mat.to_sparse() ] * 1300) + + assert _equal(res, truth).all() -- 2.26.2 From f8a02191cb98a2e68853290600e16be3daa5310d Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 4 Aug 2020 15:10:27 +0200 Subject: [PATCH 178/227] _clear_adjacency_matrix_except_rows() might yet be fast enough on the GPU. --- src/triacontagon/util.py | 20 ++++++++++++++++++-- tests/triacontagon/test_util.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/triacontagon/util.py b/src/triacontagon/util.py index 2367b06..e6fefc8 100644 --- a/src/triacontagon/util.py +++ b/src/triacontagon/util.py @@ -63,7 +63,7 @@ def _clear_adjacency_matrix_except_rows(adjacency_matrix: torch.Tensor, rows = torch.cat(rows) print('cat took:', time.time() - t) # print('rows:', rows) - rows = set(rows.tolist()) + # rows = set(rows.tolist()) # print('rows:', rows) t = time.time() @@ -71,7 +71,23 @@ def _clear_adjacency_matrix_except_rows(adjacency_matrix: torch.Tensor, indices = adj_mat.indices() values = adj_mat.values() print('indices[0]:', indices[0]) - print('indices[0][1]:', indices[0][1], indices[0][1] in rows) + # print('indices[0][1]:', indices[0][1], indices[0][1] in rows) + + lookup = torch.zeros(row_vertex_count * num_relation_types, + dtype=torch.uint8, device=adj_mat.device) + lookup[rows] = 1 + values = values * lookup[indices[0]] + mask = torch.nonzero(values > 0, as_tuple=True)[0] + indices = indices[:, mask] + values = values[mask] + + res = _sparse_coo_tensor(indices, values, adjacency_matrix.shape) + # res = res.coalesce() + print('res:', res) + print('"index_select()" took:', time.time() - t) + + return res + selection = torch.tensor([ (idx.item() in rows) for idx in indices[0] ]) # print('selection:', selection) selection = torch.nonzero(selection, as_tuple=True)[0] diff --git a/tests/triacontagon/test_util.py b/tests/triacontagon/test_util.py index 5937535..e4f7f9d 100644 --- a/tests/triacontagon/test_util.py +++ b/tests/triacontagon/test_util.py @@ -78,6 +78,8 @@ def test_clear_adjacency_matrix_except_rows_03(): def test_clear_adjacency_matrix_except_rows_04(): adj_mat = (torch.rand(2000, 2000) < 0.001).to(torch.uint8) + print('adj_mat.to_sparse():', adj_mat.to_sparse()) + t = time.time() res = _sparse_diag_cat([ adj_mat.to_sparse() ] * 1300) print('_sparse_diag_cat() took:', time.time() - t) @@ -93,3 +95,29 @@ def test_clear_adjacency_matrix_except_rows_04(): truth = _sparse_diag_cat([ adj_mat.to_sparse() ] * 1300) assert _equal(res, truth).all() + + +def test_clear_adjacency_matrix_except_rows_05(): + if torch.cuda.device_count() == 0: + pytest.skip('Test requires CUDA') + + device = torch.device('cuda:0') + adj_mat = (torch.rand(2000, 2000) < 0.001).to(torch.uint8).to(device) + + print('adj_mat.to_sparse():', adj_mat.to_sparse()) + + t = time.time() + res = _sparse_diag_cat([ adj_mat.to_sparse() ] * 1300) + print('_sparse_diag_cat() took:', time.time() - t) + + rows = torch.tensor(list(range(512)), device=device) + + t = time.time() + res = _clear_adjacency_matrix_except_rows(res, rows, + 2000, 1300) + print('_clear_adjacency_matrix_except_rows() took:', time.time() - t) + + adj_mat[512:] = torch.zeros(2000) + truth = _sparse_diag_cat([ adj_mat.to_sparse() ] * 1300) + + assert _equal(res, truth).all() -- 2.26.2 From 4ed6626e02879b742a9c0de800ff782d97a305a7 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 6 Aug 2020 10:24:06 +0200 Subject: [PATCH 179/227] Work on required vertices per layer. --- docs/required-vertices-per-layer.svg | 1463 ++++++++++++++++++++++++++ src/triacontagon/data.py | 5 +- src/triacontagon/decode.py | 10 +- src/triacontagon/model.py | 67 +- src/triacontagon/util.py | 4 +- tests/triacontagon/test_model.py | 40 + 6 files changed, 1568 insertions(+), 21 deletions(-) create mode 100644 docs/required-vertices-per-layer.svg create mode 100644 tests/triacontagon/test_model.py diff --git a/docs/required-vertices-per-layer.svg b/docs/required-vertices-per-layer.svg new file mode 100644 index 0000000..f7de037 --- /dev/null +++ b/docs/required-vertices-per-layer.svg @@ -0,0 +1,1463 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + 0 + + 1 + + 2 + + 4 + + 3 + + 0 + + 1 + + 2 + + 3 + + + + + + + + + + + + + + 0 + + 1 + + 2 + + 4 + + 3 + + 0 + + 1 + + 2 + + 3 + + + + + + + + + + + + + + + + 0 + + 1 + + 2 + + 4 + + 3 + + 0 + + 1 + + 2 + + 3 + + + + + + + + + + + + + + + + + + + 0 + + 1 + + 2 + + 4 + + 3 + + 0 + + 1 + + 2 + + 3 + + + + + + + + + + + + + + + + + + + + 0 + + 1 + + 2 + + 4 + + 3 + + 0 + + 1 + + 2 + + 3 + + + + + + + + + + + + + + + + + + + + + diff --git a/src/triacontagon/data.py b/src/triacontagon/data.py index 22a4c89..ba2b7f8 100644 --- a/src/triacontagon/data.py +++ b/src/triacontagon/data.py @@ -10,6 +10,7 @@ from typing import Callable, \ List import types from .util import _nonzero_sum +import torch @dataclass @@ -66,6 +67,6 @@ class Data(object): if (vertex_type_row, vertex_type_column) in self.edge_types: raise KeyError('Edge type for given combination of row and column already exists') total_connectivity = _nonzero_sum(adjacency_matrices) - self.edges_types[vertex_type_row, vertex_type_column] = \ - VertexType(name, vertex_type_row, vertex_type_column, + self.edge_types[vertex_type_row, vertex_type_column] = \ + EdgeType(name, vertex_type_row, vertex_type_column, adjacency_matrices, decoder_factory, total_connectivity) diff --git a/src/triacontagon/decode.py b/src/triacontagon/decode.py index 25ae822..d82d29d 100644 --- a/src/triacontagon/decode.py +++ b/src/triacontagon/decode.py @@ -11,7 +11,7 @@ from typing import Tuple, \ List -def dedicom_decoder(input_dim: int, num_relation_types: int) -> +def dedicom_decoder(input_dim: int, num_relation_types: int) -> \ Tuple[torch.Tensor, List[torch.Tensor]]: global_interaction = init_glorot(input_dim, input_dim) @@ -22,18 +22,18 @@ def dedicom_decoder(input_dim: int, num_relation_types: int) -> return (global_interaction, local_variation) -def dist_mult_decoder(input_dim: int, num_relation_types: int) -> +def dist_mult_decoder(input_dim: int, num_relation_types: int) -> \ Tuple[torch.Tensor, List[torch.Tensor]]: global_interaction = torch.eye(input_dim, input_dim) local_variation = [ - torch.diag(torch.flatten(init_glorot(input_dim, 1)))) \ + torch.diag(torch.flatten(init_glorot(input_dim, 1))) \ for _ in range(num_relation_types) ] return (global_interaction, local_variation) -def bilinear_decoder(input_dim: int, num_relation_types: int) -> +def bilinear_decoder(input_dim: int, num_relation_types: int) -> \ Tuple[torch.Tensor, List[torch.Tensor]]: global_interaction = torch.eye(input_dim, input_dim) @@ -44,7 +44,7 @@ def bilinear_decoder(input_dim: int, num_relation_types: int) -> return (global_interaction, local_variation) -def inner_product_decoder(input_dim: int, num_relation_types: int) -> +def inner_product_decoder(input_dim: int, num_relation_types: int) -> \ Tuple[torch.Tensor, List[torch.Tensor]]: global_interaction = torch.eye(input_dim, input_dim) diff --git a/src/triacontagon/model.py b/src/triacontagon/model.py index 1b98931..387c5e3 100644 --- a/src/triacontagon/model.py +++ b/src/triacontagon/model.py @@ -6,7 +6,8 @@ from .weights import init_glorot import types from typing import List, \ Dict, \ - Callable + Callable, \ + Tuple from .util import _sparse_coo_tensor @@ -18,6 +19,46 @@ class TrainingBatch(object): edges: torch.Tensor +def _per_layer_required_rows(data: Data, batch: TrainingBatch, + num_layers: int) -> List[List[EdgeType]]: + + Q = [ + ( batch.vertex_type_row, batch.edges[:, 0] ), + ( batch.vertex_type_column, batch.edges[:, 1] ) + ] + print('Q:', Q) + res = [] + + for _ in range(num_layers): + R = [] + required_rows = [ [] for _ in range(len(data.vertex_types)) ] + + for vertex_type, vertices in Q: + for et in data.edge_types.values(): + if et.vertex_type_row == vertex_type: + required_rows[vertex_type].append(vertices) + indices = et.total_connectivity.indices() + mask = torch.zeros(et.total_connectivity.shape[0]) + mask[vertices] = 1 + mask = torch.nonzero(mask[indices[0]], as_tuple=True)[0] + R.append((et.vertex_type_column, + indices[1, mask])) + else: + pass # required_rows[et.vertex_type_row].append(torch.zeros(0)) + + required_rows = [ torch.unique(torch.cat(x)) \ + if len(x) > 0 \ + else None \ + for x in required_rows ] + + res.append(required_rows) + Q = R + + return res + + + + class Model(torch.nn.Module): def __init__(self, data: Data, layer_dimensions: List[int], keep_prob: float, @@ -68,17 +109,16 @@ class Model(torch.nn.Module): torch.nn.Parameter(local_variation) ]) - def limit_adjacency_matrix_to_rows(self, adjacency_matrix: torch.Tensor, - rows: torch.Tensor) -> torch.Tensor: - - adj_mat = adjacency_matrix.coalesce() - adj_mat = torch.index_select(adj_mat, 0, rows) - adj_mat = adj_mat.coalesce() - indices = adj_mat.indices() - indices[0] = rows + def convolve(self, batch: TrainingBatch) -> List[torch.Tensor]: + edges = [] + cur_edges = batch.edges + for _ in range(len(self.layer_dimensions) - 1): + edges.append(cur_edges) + key = (batch.vertex_type_row, batch.vertex_type_column) + tot_conn = self.data.relation_types[key].total_connectivity + cur_edges = _edges_for_rows(tot_conn, cur_edges[:, 1]) - adj_mat = _sparse_coo_tensor(indices, adj_mat.values(), adjacency_matrix.shape) def temporary_adjacency_matrix(self, adjacency_matrix: torch.Tensor, batch: TrainingBatch, total_connectivity: torch.Tensor) -> torch.Tensor: @@ -90,12 +130,13 @@ class Model(torch.nn.Module): columns = torch.nonzero(columns) for i in range(len(self.layer_dimensions) - 1): + pass # columns = + # TODO: finish - columns = + return None - def temporary_adjacency_matrices(self, batch: TrainingBatch) -> - Dict[Tuple[int, int], List[List[torch.Tensor]]]: + def temporary_adjacency_matrices(self, batch: TrainingBatch) -> Dict[Tuple[int, int], List[List[torch.Tensor]]]: col = batch.vertex_type_column batch.edges[:, 1] diff --git a/src/triacontagon/util.py b/src/triacontagon/util.py index e6fefc8..51e6d07 100644 --- a/src/triacontagon/util.py +++ b/src/triacontagon/util.py @@ -41,7 +41,9 @@ def _nonzero_sum(adjacency_matrices: List[torch.Tensor]): indices = res.indices() res = _sparse_coo_tensor(indices, - torch.ones(indices.shape[1], dtype=torch.uint8)) + torch.ones(indices.shape[1], dtype=torch.uint8), + adjacency_matrices[0].shape) + res = res.coalesce() return res diff --git a/tests/triacontagon/test_model.py b/tests/triacontagon/test_model.py new file mode 100644 index 0000000..b887713 --- /dev/null +++ b/tests/triacontagon/test_model.py @@ -0,0 +1,40 @@ +from triacontagon.model import _per_layer_required_rows, \ + TrainingBatch +from triacontagon.decode import dedicom_decoder +from triacontagon.data import Data +import torch + + +def test_per_layer_required_rows_01(): + d = Data() + d.add_vertex_type('Gene', 4) + d.add_vertex_type('Drug', 5) + + d.add_edge_type('Gene-Gene', 0, 0, [ torch.tensor([ + [1, 0, 0, 1], + [0, 1, 1, 0], + [0, 0, 1, 0], + [0, 1, 0, 1] + ]).to_sparse() ], dedicom_decoder) + + d.add_edge_type('Gene-Drug', 0, 1, [ torch.tensor([ + [0, 1, 0, 0, 1], + [0, 0, 1, 0, 0], + [1, 0, 0, 0, 1], + [0, 0, 1, 1, 0] + ]).to_sparse() ], dedicom_decoder) + + d.add_edge_type('Drug-Drug', 1, 1, [ torch.tensor([ + [1, 0, 0, 0, 0], + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 0], + [0, 0, 0, 1, 0], + [0, 0, 0, 0, 1] + ]).to_sparse() ], dedicom_decoder) + + batch = TrainingBatch(0, 1, 0, torch.tensor([ + [0, 1] + ])) + + res = _per_layer_required_rows(d, batch, 5) + print('res:', res) -- 2.26.2 From b607a727aeff18fd8c0e8952e769f7bd3611af46 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 6 Aug 2020 11:58:10 +0200 Subject: [PATCH 180/227] Rename _per_layer_required_rows to _vertices. --- src/triacontagon/model.py | 40 -------------------------------- src/triacontagon/util.py | 38 ++++++++++++++++++++++++++++++ tests/triacontagon/test_util.py | 41 ++++++++++++++++++++++++++++++++- 3 files changed, 78 insertions(+), 41 deletions(-) diff --git a/src/triacontagon/model.py b/src/triacontagon/model.py index 387c5e3..8059a89 100644 --- a/src/triacontagon/model.py +++ b/src/triacontagon/model.py @@ -19,46 +19,6 @@ class TrainingBatch(object): edges: torch.Tensor -def _per_layer_required_rows(data: Data, batch: TrainingBatch, - num_layers: int) -> List[List[EdgeType]]: - - Q = [ - ( batch.vertex_type_row, batch.edges[:, 0] ), - ( batch.vertex_type_column, batch.edges[:, 1] ) - ] - print('Q:', Q) - res = [] - - for _ in range(num_layers): - R = [] - required_rows = [ [] for _ in range(len(data.vertex_types)) ] - - for vertex_type, vertices in Q: - for et in data.edge_types.values(): - if et.vertex_type_row == vertex_type: - required_rows[vertex_type].append(vertices) - indices = et.total_connectivity.indices() - mask = torch.zeros(et.total_connectivity.shape[0]) - mask[vertices] = 1 - mask = torch.nonzero(mask[indices[0]], as_tuple=True)[0] - R.append((et.vertex_type_column, - indices[1, mask])) - else: - pass # required_rows[et.vertex_type_row].append(torch.zeros(0)) - - required_rows = [ torch.unique(torch.cat(x)) \ - if len(x) > 0 \ - else None \ - for x in required_rows ] - - res.append(required_rows) - Q = R - - return res - - - - class Model(torch.nn.Module): def __init__(self, data: Data, layer_dimensions: List[int], keep_prob: float, diff --git a/src/triacontagon/util.py b/src/triacontagon/util.py index 51e6d07..eb31bbc 100644 --- a/src/triacontagon/util.py +++ b/src/triacontagon/util.py @@ -190,3 +190,41 @@ def _cat(matrices: List[torch.Tensor]): res = _sparse_coo_tensor(indices, values, size=(row_offset, matrices[0].shape[1])) return res + + +def _per_layer_required_vertices(data: Data, batch: TrainingBatch, + num_layers: int) -> List[List[EdgeType]]: + + Q = [ + ( batch.vertex_type_row, batch.edges[:, 0] ), + ( batch.vertex_type_column, batch.edges[:, 1] ) + ] + print('Q:', Q) + res = [] + + for _ in range(num_layers): + R = [] + required_rows = [ [] for _ in range(len(data.vertex_types)) ] + + for vertex_type, vertices in Q: + for et in data.edge_types.values(): + if et.vertex_type_row == vertex_type: + required_rows[vertex_type].append(vertices) + indices = et.total_connectivity.indices() + mask = torch.zeros(et.total_connectivity.shape[0]) + mask[vertices] = 1 + mask = torch.nonzero(mask[indices[0]], as_tuple=True)[0] + R.append((et.vertex_type_column, + indices[1, mask])) + else: + pass # required_rows[et.vertex_type_row].append(torch.zeros(0)) + + required_rows = [ torch.unique(torch.cat(x)) \ + if len(x) > 0 \ + else None \ + for x in required_rows ] + + res.append(required_rows) + Q = R + + return res diff --git a/tests/triacontagon/test_util.py b/tests/triacontagon/test_util.py index e4f7f9d..21f25ef 100644 --- a/tests/triacontagon/test_util.py +++ b/tests/triacontagon/test_util.py @@ -1,7 +1,11 @@ from triacontagon.util import \ _clear_adjacency_matrix_except_rows, \ _sparse_diag_cat, \ - _equal + _equal, \ + _per_layer_required_vertices +from triacontagon.model import TrainingBatch +from triacontagon.decode import dedicom_decoder +from triacontagon.data import Data import torch import time @@ -121,3 +125,38 @@ def test_clear_adjacency_matrix_except_rows_05(): truth = _sparse_diag_cat([ adj_mat.to_sparse() ] * 1300) assert _equal(res, truth).all() + + +def test_per_layer_required_vertices_01(): + d = Data() + d.add_vertex_type('Gene', 4) + d.add_vertex_type('Drug', 5) + + d.add_edge_type('Gene-Gene', 0, 0, [ torch.tensor([ + [1, 0, 0, 1], + [0, 1, 1, 0], + [0, 0, 1, 0], + [0, 1, 0, 1] + ]).to_sparse() ], dedicom_decoder) + + d.add_edge_type('Gene-Drug', 0, 1, [ torch.tensor([ + [0, 1, 0, 0, 1], + [0, 0, 1, 0, 0], + [1, 0, 0, 0, 1], + [0, 0, 1, 1, 0] + ]).to_sparse() ], dedicom_decoder) + + d.add_edge_type('Drug-Drug', 1, 1, [ torch.tensor([ + [1, 0, 0, 0, 0], + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 0], + [0, 0, 0, 1, 0], + [0, 0, 0, 0, 1] + ]).to_sparse() ], dedicom_decoder) + + batch = TrainingBatch(0, 1, 0, torch.tensor([ + [0, 1] + ])) + + res = _per_layer_required_vertices(d, batch, 5) + print('res:', res) -- 2.26.2 From bd8143469c54d99fbfcd7301dc3c3264d888bb8a Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 6 Aug 2020 15:12:09 +0200 Subject: [PATCH 181/227] Make work convolve() in new model. --- src/triacontagon/{ => deprecated}/fastconv.py | 0 src/triacontagon/{ => deprecated}/fastdec.py | 0 src/triacontagon/{ => deprecated}/fastloop.py | 0 .../{ => deprecated}/fastmodel.py | 0 .../{ => deprecated}/trainprep.py | 0 src/triacontagon/model.py | 126 ++++++++++++++++-- src/triacontagon/util.py | 41 +----- tests/triacontagon/test_model.py | 34 +++-- 8 files changed, 138 insertions(+), 63 deletions(-) rename src/triacontagon/{ => deprecated}/fastconv.py (100%) rename src/triacontagon/{ => deprecated}/fastdec.py (100%) rename src/triacontagon/{ => deprecated}/fastloop.py (100%) rename src/triacontagon/{ => deprecated}/fastmodel.py (100%) rename src/triacontagon/{ => deprecated}/trainprep.py (100%) diff --git a/src/triacontagon/fastconv.py b/src/triacontagon/deprecated/fastconv.py similarity index 100% rename from src/triacontagon/fastconv.py rename to src/triacontagon/deprecated/fastconv.py diff --git a/src/triacontagon/fastdec.py b/src/triacontagon/deprecated/fastdec.py similarity index 100% rename from src/triacontagon/fastdec.py rename to src/triacontagon/deprecated/fastdec.py diff --git a/src/triacontagon/fastloop.py b/src/triacontagon/deprecated/fastloop.py similarity index 100% rename from src/triacontagon/fastloop.py rename to src/triacontagon/deprecated/fastloop.py diff --git a/src/triacontagon/fastmodel.py b/src/triacontagon/deprecated/fastmodel.py similarity index 100% rename from src/triacontagon/fastmodel.py rename to src/triacontagon/deprecated/fastmodel.py diff --git a/src/triacontagon/trainprep.py b/src/triacontagon/deprecated/trainprep.py similarity index 100% rename from src/triacontagon/trainprep.py rename to src/triacontagon/deprecated/trainprep.py diff --git a/src/triacontagon/model.py b/src/triacontagon/model.py index 8059a89..49ca1bb 100644 --- a/src/triacontagon/model.py +++ b/src/triacontagon/model.py @@ -8,7 +8,12 @@ from typing import List, \ Dict, \ Callable, \ Tuple -from .util import _sparse_coo_tensor +from .util import _sparse_coo_tensor, \ + _sparse_diag_cat, \ + _mm +from .normalize import norm_adj_mat_one_node_type, \ + norm_adj_mat_two_node_types +from .dropout import dropout @dataclass @@ -19,6 +24,44 @@ class TrainingBatch(object): edges: torch.Tensor +def _per_layer_required_vertices(data: Data, batch: TrainingBatch, + num_layers: int) -> List[List[EdgeType]]: + + Q = [ + ( batch.vertex_type_row, batch.edges[:, 0] ), + ( batch.vertex_type_column, batch.edges[:, 1] ) + ] + print('Q:', Q) + res = [] + + for _ in range(num_layers): + R = [] + required_rows = [ [] for _ in range(len(data.vertex_types)) ] + + for vertex_type, vertices in Q: + for et in data.edge_types.values(): + if et.vertex_type_row == vertex_type: + required_rows[vertex_type].append(vertices) + indices = et.total_connectivity.indices() + mask = torch.zeros(et.total_connectivity.shape[0]) + mask[vertices] = 1 + mask = torch.nonzero(mask[indices[0]], as_tuple=True)[0] + R.append((et.vertex_type_column, + indices[1, mask])) + else: + pass # required_rows[et.vertex_type_row].append(torch.zeros(0)) + + required_rows = [ torch.unique(torch.cat(x)) \ + if len(x) > 0 \ + else None \ + for x in required_rows ] + + res.append(required_rows) + Q = R + + return res + + class Model(torch.nn.Module): def __init__(self, data: Data, layer_dimensions: List[int], keep_prob: float, @@ -30,11 +73,11 @@ class Model(torch.nn.Module): if not isinstance(data, Data): raise TypeError('data must be an instance of Data') - if not isinstance(conv_activation, types.FunctionType): - raise TypeError('conv_activation must be a function') + if not callable(conv_activation): + raise TypeError('conv_activation must be callable') - if not isinstance(dec_activation, types.FunctionType): - raise TypeError('dec_activation must be a function') + if not callable(dec_activation): + raise TypeError('dec_activation must be callable') self.data = data self.layer_dimensions = list(layer_dimensions) @@ -42,35 +85,90 @@ class Model(torch.nn.Module): self.conv_activation = conv_activation self.dec_activation = dec_activation + self.adj_matrices = None self.conv_weights = None self.dec_weights = None self.build() def build(self) -> None: + self.adj_matrices = torch.nn.ParameterDict() + for _, et in self.data.edge_types.items(): + adj_matrices = [ + norm_adj_mat_one_node_type(x) \ + if et.vertex_type_row == et.vertex_type_column \ + else norm_adj_mat_two_node_types(x) \ + for x in et.adjacency_matrices + ] + adj_matrices = _sparse_diag_cat(et.adjacency_matrices) + print('adj_matrices:', adj_matrices) + self.adj_matrices['%d-%d' % (et.vertex_type_row, et.vertex_type_column)] = \ + torch.nn.Parameter(adj_matrices, requires_grad=False) + self.conv_weights = torch.nn.ParameterDict() for i in range(len(self.layer_dimensions) - 1): in_dimension = self.layer_dimensions[i] out_dimension = self.layer_dimensions[i + 1] for _, et in self.data.edge_types.items(): - weight = init_glorot(in_dimension, out_dimension) - self.conv_weights[et.vertex_type_row, et.vertex_type_column, i] = \ - torch.nn.Parameter(weight) + weights = [ init_glorot(in_dimension, out_dimension) \ + for _ in range(len(et.adjacency_matrices)) ] + weights = torch.cat(weights, dim=1) + self.conv_weights['%d-%d-%d' % (et.vertex_type_row, et.vertex_type_column, i)] = \ + torch.nn.Parameter(weights) self.dec_weights = torch.nn.ParameterDict() for _, et in self.data.edge_types.items(): global_interaction, local_variation = \ et.decoder_factory(self.layer_dimensions[-1], len(et.adjacency_matrices)) - self.dec_weights[et.vertex_type_row, et.vertex_type_column] = \ - torch.nn.ParameterList([ - torch.nn.Parameter(global_interaction), - torch.nn.Parameter(local_variation) - ]) + self.dec_weights['%d-%d-global-interaction' % (et.vertex_type_row, et.vertex_type_column)] = \ + torch.nn.Parameter(global_interaction) + for i in range(len(local_variation)): + self.dec_weights['%d-%d-local-variation-%d' % (et.vertex_type_row, et.vertex_type_column, i)] = \ + torch.nn.Parameter(local_variation[i]) + + def convolve(self, in_layer_repr: List[torch.Tensor]) -> \ + List[torch.Tensor]: - def convolve(self, batch: TrainingBatch) -> List[torch.Tensor]: + cur_layer_repr = in_layer_repr + next_layer_repr = [ None ] * len(self.data.vertex_types) + + for i in range(len(self.layer_dimensions) - 1): + for _, et in self.data.edge_types.items(): + vt_row, vt_col = et.vertex_type_row, et.vertex_type_column + adj_matrices = self.adj_matrices['%d-%d' % (vt_row, vt_col)] + conv_weights = self.conv_weights['%d-%d-%d' % (vt_row, vt_col, i)] + + num_relation_types = len(et.adjacency_matrices) + x = cur_layer_repr[vt_col] + if self.keep_prob != 1: + x = dropout(x, self.keep_prob) + + print('a, Layer:', i, 'x.shape:', x.shape) + + x = _mm(x, conv_weights) + x = torch.split(x, + x.shape[1] // num_relation_types, + dim=1) + x = torch.cat(x) + x = _mm(adj_matrices, x) + x = x.view(num_relation_types, + self.data.vertex_types[vt_row].count, + self.layer_dimensions[i + 1]) + + print('b, Layer:', i, 'x.shape:', x.shape) + + x = x.sum(dim=0) + x = torch.nn.functional.normalize(x, p=2, dim=1) + x = self.conv_activation(x) + + next_layer_repr[vt_row] = x + cur_layer_repr = next_layer_repr + return next_layer_repr + + def convolve_old(self, batch: TrainingBatch) -> List[torch.Tensor]: edges = [] cur_edges = batch.edges for _ in range(len(self.layer_dimensions) - 1): diff --git a/src/triacontagon/util.py b/src/triacontagon/util.py index eb31bbc..134b64c 100644 --- a/src/triacontagon/util.py +++ b/src/triacontagon/util.py @@ -192,39 +192,8 @@ def _cat(matrices: List[torch.Tensor]): return res -def _per_layer_required_vertices(data: Data, batch: TrainingBatch, - num_layers: int) -> List[List[EdgeType]]: - - Q = [ - ( batch.vertex_type_row, batch.edges[:, 0] ), - ( batch.vertex_type_column, batch.edges[:, 1] ) - ] - print('Q:', Q) - res = [] - - for _ in range(num_layers): - R = [] - required_rows = [ [] for _ in range(len(data.vertex_types)) ] - - for vertex_type, vertices in Q: - for et in data.edge_types.values(): - if et.vertex_type_row == vertex_type: - required_rows[vertex_type].append(vertices) - indices = et.total_connectivity.indices() - mask = torch.zeros(et.total_connectivity.shape[0]) - mask[vertices] = 1 - mask = torch.nonzero(mask[indices[0]], as_tuple=True)[0] - R.append((et.vertex_type_column, - indices[1, mask])) - else: - pass # required_rows[et.vertex_type_row].append(torch.zeros(0)) - - required_rows = [ torch.unique(torch.cat(x)) \ - if len(x) > 0 \ - else None \ - for x in required_rows ] - - res.append(required_rows) - Q = R - - return res +def _mm(a: torch.Tensor, b: torch.Tensor): + if a.is_sparse: + return torch.sparse.mm(a, b) + else: + return torch.mm(a, b) diff --git a/tests/triacontagon/test_model.py b/tests/triacontagon/test_model.py index b887713..c943d8e 100644 --- a/tests/triacontagon/test_model.py +++ b/tests/triacontagon/test_model.py @@ -1,11 +1,10 @@ -from triacontagon.model import _per_layer_required_rows, \ - TrainingBatch -from triacontagon.decode import dedicom_decoder -from triacontagon.data import Data import torch +from triacontagon.model import Model +from triacontagon.data import Data +from triacontagon.decode import dedicom_decoder -def test_per_layer_required_rows_01(): +def test_model_convolve_01(): d = Data() d.add_vertex_type('Gene', 4) d.add_vertex_type('Drug', 5) @@ -15,14 +14,14 @@ def test_per_layer_required_rows_01(): [0, 1, 1, 0], [0, 0, 1, 0], [0, 1, 0, 1] - ]).to_sparse() ], dedicom_decoder) + ], dtype=torch.float).to_sparse() ], dedicom_decoder) d.add_edge_type('Gene-Drug', 0, 1, [ torch.tensor([ [0, 1, 0, 0, 1], [0, 0, 1, 0, 0], [1, 0, 0, 0, 1], [0, 0, 1, 1, 0] - ]).to_sparse() ], dedicom_decoder) + ], dtype=torch.float).to_sparse() ], dedicom_decoder) d.add_edge_type('Drug-Drug', 1, 1, [ torch.tensor([ [1, 0, 0, 0, 0], @@ -30,11 +29,20 @@ def test_per_layer_required_rows_01(): [0, 0, 1, 0, 0], [0, 0, 0, 1, 0], [0, 0, 0, 0, 1] - ]).to_sparse() ], dedicom_decoder) + ], dtype=torch.float).to_sparse() ], dedicom_decoder) + + model = Model(d, [9, 32, 64], keep_prob=1.0, + conv_activation = torch.sigmoid, + dec_activation = torch.sigmoid) + + repr_1 = torch.eye(9) + repr_1[4:, 4:] = 0 + repr_2 = torch.eye(9) + repr_2[:4, :4] = 0 - batch = TrainingBatch(0, 1, 0, torch.tensor([ - [0, 1] - ])) + in_layer_repr = [ + repr_1[:4, :].to_sparse(), + repr_2[4:, :].to_sparse() + ] - res = _per_layer_required_rows(d, batch, 5) - print('res:', res) + _ = model.convolve(in_layer_repr) -- 2.26.2 From 82d5c06eee4c68028e6dba621b42bd8ce809527d Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 6 Aug 2020 15:25:50 +0200 Subject: [PATCH 182/227] Fix aggregation of representations from different edge types. --- src/triacontagon/model.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/triacontagon/model.py b/src/triacontagon/model.py index 49ca1bb..49d37bd 100644 --- a/src/triacontagon/model.py +++ b/src/triacontagon/model.py @@ -133,9 +133,10 @@ class Model(torch.nn.Module): List[torch.Tensor]: cur_layer_repr = in_layer_repr - next_layer_repr = [ None ] * len(self.data.vertex_types) - + for i in range(len(self.layer_dimensions) - 1): + next_layer_repr = [ [] for _ in range(len(self.data.vertex_types)) ] + for _, et in self.data.edge_types.items(): vt_row, vt_col = et.vertex_type_row, et.vertex_type_column adj_matrices = self.adj_matrices['%d-%d' % (vt_row, vt_col)] @@ -158,13 +159,18 @@ class Model(torch.nn.Module): self.data.vertex_types[vt_row].count, self.layer_dimensions[i + 1]) - print('b, Layer:', i, 'x.shape:', x.shape) + print('b, Layer:', i, 'vt_row:', vt_row, 'x.shape:', x.shape) x = x.sum(dim=0) x = torch.nn.functional.normalize(x, p=2, dim=1) - x = self.conv_activation(x) + # x = self.rel_activation(x) + print('c, Layer:', i, 'vt_row:', vt_row, 'x.shape:', x.shape) + + next_layer_repr[vt_row].append(x) + + next_layer_repr = [ self.conv_activation(sum(x)) \ + for x in next_layer_repr ] - next_layer_repr[vt_row] = x cur_layer_repr = next_layer_repr return next_layer_repr -- 2.26.2 From b1a5f0d0ee311be9e3de8592ceff5ee6ba3c7cb6 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 7 Aug 2020 09:56:50 +0200 Subject: [PATCH 183/227] Add common_one_hot_encoding(). --- src/triacontagon/{ => deprecated}/input.py | 0 src/triacontagon/dropout.py | 2 ++ src/triacontagon/model.py | 35 ++++++++++++++++--- src/triacontagon/util.py | 22 ++++++++++++ tests/triacontagon/test_model.py | 39 +++++++++++++++++++++- tests/triacontagon/test_util.py | 39 ++++------------------ 6 files changed, 99 insertions(+), 38 deletions(-) rename src/triacontagon/{ => deprecated}/input.py (100%) diff --git a/src/triacontagon/input.py b/src/triacontagon/deprecated/input.py similarity index 100% rename from src/triacontagon/input.py rename to src/triacontagon/deprecated/input.py diff --git a/src/triacontagon/dropout.py b/src/triacontagon/dropout.py index 63cfb58..2fb8728 100644 --- a/src/triacontagon/dropout.py +++ b/src/triacontagon/dropout.py @@ -36,6 +36,8 @@ def dropout_dense(x, keep_prob): def dropout(x, keep_prob): + if keep_prob == 1: + return x if x.is_sparse: return dropout_sparse(x, keep_prob) else: diff --git a/src/triacontagon/model.py b/src/triacontagon/model.py index 49d37bd..58f635d 100644 --- a/src/triacontagon/model.py +++ b/src/triacontagon/model.py @@ -22,6 +22,7 @@ class TrainingBatch(object): vertex_type_column: int relation_type_index: int edges: torch.Tensor + target_values: torch.Tensor def _per_layer_required_vertices(data: Data, batch: TrainingBatch, @@ -133,7 +134,7 @@ class Model(torch.nn.Module): List[torch.Tensor]: cur_layer_repr = in_layer_repr - + for i in range(len(self.layer_dimensions) - 1): next_layer_repr = [ [] for _ in range(len(self.data.vertex_types)) ] @@ -147,7 +148,7 @@ class Model(torch.nn.Module): if self.keep_prob != 1: x = dropout(x, self.keep_prob) - print('a, Layer:', i, 'x.shape:', x.shape) + # print('a, Layer:', i, 'x.shape:', x.shape) x = _mm(x, conv_weights) x = torch.split(x, @@ -159,12 +160,12 @@ class Model(torch.nn.Module): self.data.vertex_types[vt_row].count, self.layer_dimensions[i + 1]) - print('b, Layer:', i, 'vt_row:', vt_row, 'x.shape:', x.shape) + # print('b, Layer:', i, 'vt_row:', vt_row, 'x.shape:', x.shape) x = x.sum(dim=0) x = torch.nn.functional.normalize(x, p=2, dim=1) # x = self.rel_activation(x) - print('c, Layer:', i, 'vt_row:', vt_row, 'x.shape:', x.shape) + # print('c, Layer:', i, 'vt_row:', vt_row, 'x.shape:', x.shape) next_layer_repr[vt_row].append(x) @@ -174,6 +175,32 @@ class Model(torch.nn.Module): cur_layer_repr = next_layer_repr return next_layer_repr + def decode(self, last_layer_repr: List[torch.Tensor], + batch: TrainingBatch) -> torch.Tensor: + + vt_row = batch.vertex_type_row + vt_col = batch.vertex_type_column + rel_idx = batch.relation_type_index + global_interaction = \ + self.dec_weights['%d-%d-global-interaction'] % (vt_row, vt_col) + local_variation = \ + self.dec_weights['%d-%d-local-variation-%d'] % (vt_row, vt_col, rel_idx) + + in_row = dropout(last_layer_repr[vt_row], self.keep_prob) + in_col = dropout(last_layer_repr[vt_col], self.keep_prob) + + x = torch.mm(in_row, local_variation) + x = torch.mm(x, global_interaction) + x = torch.mm(x, local_variation) + x = torch.bmm(x.view(x.shape[0], 1, x.shape[1]), + in_col.view(in_col.shape[0], in_col.shape[1], 1)) + x = torch.flatten(x) + + x = self.dec_activation(x) + + return x + + def convolve_old(self, batch: TrainingBatch) -> List[torch.Tensor]: edges = [] cur_edges = batch.edges diff --git a/src/triacontagon/util.py b/src/triacontagon/util.py index 134b64c..8d8bad4 100644 --- a/src/triacontagon/util.py +++ b/src/triacontagon/util.py @@ -197,3 +197,25 @@ def _mm(a: torch.Tensor, b: torch.Tensor): return torch.sparse.mm(a, b) else: return torch.mm(a, b) + + +def common_one_hot_encoding(vertex_type_counts: List[int]) -> \ + List[torch.Tensor]: + + tot = sum(vertex_type_counts) + # indices = torch.cat([ torch.arange(tot).view(1, -1) ] * 2, dim=0) + # print('indices.shape:', indices.shape) + ofs = 0 + res = [] + + for cnt in vertex_type_counts: + ind = torch.cat([ + torch.arange(cnt).view(1, -1), + torch.arange(ofs, ofs+cnt).view(1, -1) + ]) + val = torch.ones(cnt) + x = _sparse_coo_tensor(ind, val, size=(cnt, tot)) + res.append(x) + ofs += cnt + + return res diff --git a/tests/triacontagon/test_model.py b/tests/triacontagon/test_model.py index c943d8e..20656c9 100644 --- a/tests/triacontagon/test_model.py +++ b/tests/triacontagon/test_model.py @@ -1,9 +1,46 @@ import torch -from triacontagon.model import Model +from triacontagon.model import Model, \ + TrainingBatch, \ + _per_layer_required_vertices from triacontagon.data import Data from triacontagon.decode import dedicom_decoder +def test_per_layer_required_vertices_01(): + d = Data() + d.add_vertex_type('Gene', 4) + d.add_vertex_type('Drug', 5) + + d.add_edge_type('Gene-Gene', 0, 0, [ torch.tensor([ + [1, 0, 0, 1], + [0, 1, 1, 0], + [0, 0, 1, 0], + [0, 1, 0, 1] + ]).to_sparse() ], dedicom_decoder) + + d.add_edge_type('Gene-Drug', 0, 1, [ torch.tensor([ + [0, 1, 0, 0, 1], + [0, 0, 1, 0, 0], + [1, 0, 0, 0, 1], + [0, 0, 1, 1, 0] + ]).to_sparse() ], dedicom_decoder) + + d.add_edge_type('Drug-Drug', 1, 1, [ torch.tensor([ + [1, 0, 0, 0, 0], + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 0], + [0, 0, 0, 1, 0], + [0, 0, 0, 0, 1] + ]).to_sparse() ], dedicom_decoder) + + batch = TrainingBatch(0, 1, 0, torch.tensor([ + [0, 1] + ])) + + res = _per_layer_required_vertices(d, batch, 5) + print('res:', res) + + def test_model_convolve_01(): d = Data() d.add_vertex_type('Gene', 4) diff --git a/tests/triacontagon/test_util.py b/tests/triacontagon/test_util.py index 21f25ef..3593d1d 100644 --- a/tests/triacontagon/test_util.py +++ b/tests/triacontagon/test_util.py @@ -2,7 +2,7 @@ from triacontagon.util import \ _clear_adjacency_matrix_except_rows, \ _sparse_diag_cat, \ _equal, \ - _per_layer_required_vertices + common_one_hot_encoding from triacontagon.model import TrainingBatch from triacontagon.decode import dedicom_decoder from triacontagon.data import Data @@ -127,36 +127,9 @@ def test_clear_adjacency_matrix_except_rows_05(): assert _equal(res, truth).all() -def test_per_layer_required_vertices_01(): - d = Data() - d.add_vertex_type('Gene', 4) - d.add_vertex_type('Drug', 5) +def test_common_one_hot_encoding_01(): + in_repr = common_one_hot_encoding([2000, 200]) - d.add_edge_type('Gene-Gene', 0, 0, [ torch.tensor([ - [1, 0, 0, 1], - [0, 1, 1, 0], - [0, 0, 1, 0], - [0, 1, 0, 1] - ]).to_sparse() ], dedicom_decoder) - - d.add_edge_type('Gene-Drug', 0, 1, [ torch.tensor([ - [0, 1, 0, 0, 1], - [0, 0, 1, 0, 0], - [1, 0, 0, 0, 1], - [0, 0, 1, 1, 0] - ]).to_sparse() ], dedicom_decoder) - - d.add_edge_type('Drug-Drug', 1, 1, [ torch.tensor([ - [1, 0, 0, 0, 0], - [0, 1, 0, 0, 0], - [0, 0, 1, 0, 0], - [0, 0, 0, 1, 0], - [0, 0, 0, 0, 1] - ]).to_sparse() ], dedicom_decoder) - - batch = TrainingBatch(0, 1, 0, torch.tensor([ - [0, 1] - ])) - - res = _per_layer_required_vertices(d, batch, 5) - print('res:', res) + ref = torch.eye(2200) + assert torch.all(in_repr[0].to_dense() == ref[:2000, :]) + assert torch.all(in_repr[1].to_dense() == ref[2000:, :]) -- 2.26.2 From 7ea82a626756080c0c8ac5a32ead0c1f58341426 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 7 Aug 2020 11:21:58 +0200 Subject: [PATCH 184/227] Add test_model_decode_01(). --- src/triacontagon/model.py | 27 +++++++++++++++++++++------ src/triacontagon/util.py | 26 ++++++++++++++++++++++++++ tests/triacontagon/test_model.py | 26 ++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/src/triacontagon/model.py b/src/triacontagon/model.py index 58f635d..1a41b4d 100644 --- a/src/triacontagon/model.py +++ b/src/triacontagon/model.py @@ -173,7 +173,7 @@ class Model(torch.nn.Module): for x in next_layer_repr ] cur_layer_repr = next_layer_repr - return next_layer_repr + return cur_layer_repr def decode(self, last_layer_repr: List[torch.Tensor], batch: TrainingBatch) -> torch.Tensor: @@ -182,12 +182,27 @@ class Model(torch.nn.Module): vt_col = batch.vertex_type_column rel_idx = batch.relation_type_index global_interaction = \ - self.dec_weights['%d-%d-global-interaction'] % (vt_row, vt_col) + self.dec_weights['%d-%d-global-interaction' % (vt_row, vt_col)] local_variation = \ - self.dec_weights['%d-%d-local-variation-%d'] % (vt_row, vt_col, rel_idx) + self.dec_weights['%d-%d-local-variation-%d' % (vt_row, vt_col, rel_idx)] - in_row = dropout(last_layer_repr[vt_row], self.keep_prob) - in_col = dropout(last_layer_repr[vt_col], self.keep_prob) + in_row = last_layer_repr[vt_row] + in_col = last_layer_repr[vt_col] + + if in_row.is_sparse or in_col.is_sparse: + raise ValueError('Inputs to Model.decode() must be dense') + + in_row = in_row[batch.edges[:, 0]] + in_col = in_col[batch.edges[:, 1]] + + in_row = dropout(in_row, self.keep_prob) + in_col = dropout(in_col, self.keep_prob) + + # in_row = in_row.to_dense() + # in_col = in_col.to_dense() + + print('in_row.is_sparse:', in_row.is_sparse) + print('in_col.is_sparse:', in_col.is_sparse) x = torch.mm(in_row, local_variation) x = torch.mm(x, global_interaction) @@ -197,7 +212,7 @@ class Model(torch.nn.Module): x = torch.flatten(x) x = self.dec_activation(x) - + return x diff --git a/src/triacontagon/util.py b/src/triacontagon/util.py index 8d8bad4..f268f44 100644 --- a/src/triacontagon/util.py +++ b/src/triacontagon/util.py @@ -199,6 +199,32 @@ def _mm(a: torch.Tensor, b: torch.Tensor): return torch.mm(a, b) +def _select_rows(a: torch.Tensor, rows: torch.Tensor): + if not a.is_sparse: + return a[rows] + + indices = a.indices() + values = a.values() + + mask = torch.zeros(a.shape[0]) + mask[rows] = 1 + if mask.sum() != len(rows): + raise ValueError('Rows must be unique') + mask = mask[indices[0]] + mask = torch.nonzero(mask, as_tuple=True)[0] + + new_rows[rows] = torch.arange(len(rows)) + new_rows = new_rows[indices[0]] + + indices = indices[:, mask] + indices[0] = new_rows + values = values[mask] + + res = _sparse_coo_tensor(indices, values, + size=(len(rows), a.shape[1])) + return res + + def common_one_hot_encoding(vertex_type_counts: List[int]) -> \ List[torch.Tensor]: diff --git a/tests/triacontagon/test_model.py b/tests/triacontagon/test_model.py index 20656c9..0f57578 100644 --- a/tests/triacontagon/test_model.py +++ b/tests/triacontagon/test_model.py @@ -4,6 +4,7 @@ from triacontagon.model import Model, \ _per_layer_required_vertices from triacontagon.data import Data from triacontagon.decode import dedicom_decoder +from triacontagon.util import common_one_hot_encoding def test_per_layer_required_vertices_01(): @@ -83,3 +84,28 @@ def test_model_convolve_01(): ] _ = model.convolve(in_layer_repr) + + +def test_model_decode_01(): + d = Data() + d.add_vertex_type('Gene', 100) + + d.add_edge_type('Gene-Gene', 0, 0, [ + torch.rand(100, 100).round().to_sparse() + ], dedicom_decoder) + + b = TrainingBatch(0, 0, 0, torch.tensor([ + [0, 1], + [10, 51], + [50, 60], + [70, 90], + [98, 99] + ]), torch.ones(5)) + + in_repr = common_one_hot_encoding([100]) + + in_repr = [ in_repr[0].to_dense() ] + + m = Model(d, [100], 1.0, torch.sigmoid, torch.sigmoid) + + _ = m.decode(in_repr, b) -- 2.26.2 From e557b18762f02f04ab8c0b3ab9ccd469bf7987fb Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 7 Aug 2020 14:50:32 +0200 Subject: [PATCH 185/227] Add trainprep and negative_sample_(adj_mat|data)(). --- src/triacontagon/sampling.py | 73 ++++++++++++++++++++++++++++- src/triacontagon/trainprep.py | 59 +++++++++++++++++++++++ tests/triacontagon/test_sampling.py | 38 +++++++++++++++ 3 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 src/triacontagon/trainprep.py create mode 100644 tests/triacontagon/test_sampling.py diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py index 7c55944..60b7647 100644 --- a/src/triacontagon/sampling.py +++ b/src/triacontagon/sampling.py @@ -8,7 +8,10 @@ import numpy as np import torch import torch.utils.data from typing import List, \ - Union + Union, \ + Tuple +from .data import Data, \ + EdgeType def fixed_unigram_candidate_sampler( @@ -24,7 +27,7 @@ def fixed_unigram_candidate_sampler( if len(true_classes.shape) != 2: raise ValueError('true_classes must be a 2D matrix with shape (num_samples, num_true)') - + num_samples = true_classes.shape[0] unigrams = np.array(unigrams) if distortion != 1.: @@ -40,8 +43,74 @@ def fixed_unigram_candidate_sampler( # print('candidates:', candidates) # print('true_classes:', true_classes[indices, :]) result[indices] = candidates.T + # print('result:', result) mask = (candidates == true_classes[indices, :]) mask = mask.sum(1).astype(np.bool) # print('mask:', mask) indices = indices[mask] + # result[indices] = 0 return torch.tensor(result) + + +def get_edges_and_degrees(adj_mat: torch.Tensor) -> \ + Tuple[torch.Tensor, torch.Tensor]: + + if adj_mat.is_sparse: + adj_mat = adj_mat.coalesce() + degrees = torch.zeros(adj_mat.shape[1], dtype=torch.int64, + device=adj_mat.device) + degrees = degrees.index_add(0, adj_mat.indices()[1], + torch.ones(adj_mat.indices().shape[1], dtype=torch.int64, + device=adj_mat.device)) + edges_pos = adj_mat.indices().transpose(0, 1) + else: + degrees = adj_mat.sum(0) + edges_pos = torch.nonzero(adj_mat, as_tuple=False) + return edges_pos, degrees + + +def negative_sample_adj_mat(adj_mat: torch.Tensor) -> torch.Tensor: + if not isinstance(adj_mat, torch.Tensor): + raise ValueError('adj_mat must be a torch.Tensor, got: %s' % adj_mat.__class__.__name__) + + edges_pos, degrees = get_edges_and_degrees(adj_mat) + + neg_neighbors = fixed_unigram_candidate_sampler( + edges_pos[:, 1].view(-1, 1), degrees, 0.75).to(adj_mat.device) + edges_neg = torch.cat([ edges_pos[:, 0].view(-1, 1), + neg_neighbors.view(-1, 1) ], 1) + + adj_mat_neg = torch.sparse_coo_tensor(indices = edges_neg.transpose(0, 1), + values=torch.ones(len(edges_neg)), size=adj_mat.shape, + dtype=adj_mat.dtype, device=adj_mat.device) + + adj_mat_neg = adj_mat_neg.coalesce() + indices = adj_mat_neg.indices() + adj_mat_neg = torch.sparse_coo_tensor(indices, + torch.ones(indices.shape[1]), adj_mat.shape, + dtype=adj_mat.dtype, device=adj_mat.device) + + adj_mat_neg = adj_mat_neg.coalesce() + + return adj_mat_neg + + +def negative_sample_data(data: Data) -> Data: + new_edge_types = {} + res = Data() + for vt in data.vertex_types: + res.add_vertex_type(vt.name, vt.count) + for key, et in data.edge_types.items(): + adjacency_matrices_neg = [] + for adj_mat in et.adjacency_matrices: + adj_mat_neg = negative_sample_adj_mat(adj_mat) + adjacency_matrices_neg.append(adj_mat_neg) + res.add_edge_type(et.name, + et.vertex_type_row, et.vertex_type_column, + adjacency_matrices_neg, et.decoder_factory) + #new_et = EdgeType(et.name, et.vertex_type_row, + # et.vertex_type_column, adjacency_matrices_neg, + # et.decoder_factory, et.total_connectivity) + #new_edge_types[key] = new_et + #res = Data(data.vertex_types, new_edge_types) + return res diff --git a/src/triacontagon/trainprep.py b/src/triacontagon/trainprep.py new file mode 100644 index 0000000..dd7a12c --- /dev/null +++ b/src/triacontagon/trainprep.py @@ -0,0 +1,59 @@ +from .data import Data, \ + TrainingBatch, \ + EdgeType +from typing import Tuple +from .util import _sparse_coo_tensor + + +def split_adj_mat(adj_mat: torch.Tensor, ratios: List[float]): + indices = adj_mat.indices() + values = adj_mat.values() + + order = torch.randperm(indices.shape[1]) + + indices = indices[:, order] + values = values[order] + + ofs = 0 + res = [] + for r in ratios: + cnt = r * len(values) + ind = indices[:, ofs:ofs+cnt] + val = values[ofs:ofs+cnt] + res.append(_sparse_coo_tensor(ind, val, adj_mat.shape)) + ofs += cnt + + return res + + +def split_edge_type(et: EdgeType, ratios: Tuple[float, float, float]): + res = [ [] for _ in range(len(et.adjacency_matrices)) ] + + for adj_mat in et.adjacency_matrices: + for i, new_adj_mat in enumerate(split_adj_mat(adj_mat, ratios)): + res[i].append(new_adj_mat) + + return res + + +def split_data(data: Data, + ratios: List[float]): + + if not isinstance(data, Data): + raise TypeError('data must be an instance of Data') + + ratios = list(ratios) + + if sum(ratios) != 1: + raise ValueError('ratios must sum to 1') + + res = [ {} for _ in range(len(ratios)) ] + + for key, et in data.edge_types: + for i, new_et in enumerate(split_edge_type(et, ratios)): + res[i][key] = new_et + + res = [ Data(data.vertex_types, new_edge_types) \ + for new_edge_types in res ] + + return res diff --git a/tests/triacontagon/test_sampling.py b/tests/triacontagon/test_sampling.py new file mode 100644 index 0000000..6d7a155 --- /dev/null +++ b/tests/triacontagon/test_sampling.py @@ -0,0 +1,38 @@ +from triacontagon.data import Data +from triacontagon.sampling import negative_sample_adj_mat, \ + negative_sample_data +from triacontagon.decode import dedicom_decoder +import torch + + +def test_negative_sample_adj_mat_01(): + adj_mat = torch.tensor([ + [0, 1, 0, 1, 0], + [0, 0, 0, 0, 1], + [1, 1, 0, 0, 0], + [0, 0, 1, 0, 1], + [0, 1, 0, 0, 0] + ]) + + print('adj_mat:', adj_mat) + + adj_mat_neg = negative_sample_adj_mat(adj_mat) + + print('adj_mat_neg:', adj_mat_neg.to_dense()) + + +def test_negative_sample_data_01(): + d = Data() + d.add_vertex_type('Gene', 5) + + d.add_edge_type('Gene-Gene', 0, 0, [ + torch.tensor([ + [0, 1, 0, 1, 0], + [0, 0, 0, 0, 1], + [1, 1, 0, 0, 0], + [0, 0, 1, 0, 1], + [0, 1, 0, 0, 0] + ], dtype=torch.float).to_sparse() + ], dedicom_decoder) + + d_neg = negative_sample_data(d) -- 2.26.2 From 4ed3715b832cb09d320613effd9784a9263f5177 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 7 Aug 2020 16:28:05 +0200 Subject: [PATCH 186/227] Add cumcount(). --- src/triacontagon/cumcount.py | 22 +++++++++ src/triacontagon/deprecated/fastconv.py | 64 +------------------------ src/triacontagon/sampling.py | 52 +++++++++++++++++++- tests/triacontagon/test_sampling.py | 37 +++++++++++++- 4 files changed, 110 insertions(+), 65 deletions(-) create mode 100644 src/triacontagon/cumcount.py diff --git a/src/triacontagon/cumcount.py b/src/triacontagon/cumcount.py new file mode 100644 index 0000000..ba1f23f --- /dev/null +++ b/src/triacontagon/cumcount.py @@ -0,0 +1,22 @@ +import numpy as np + + +def dfill(a): + n = a.size + b = np.concatenate([[0], np.where(a[:-1] != a[1:])[0] + 1, [n]]) + return np.arange(n)[b[:-1]].repeat(np.diff(b)) + + +def argunsort(s): + n = s.size + u = np.empty(n, dtype=np.int64) + u[s] = np.arange(n) + return u + + +def cumcount(a): + n = a.size + s = a.argsort(kind='mergesort') + i = argunsort(s) + b = a[s] + return (np.arange(n) - dfill(b))[i] diff --git a/src/triacontagon/deprecated/fastconv.py b/src/triacontagon/deprecated/fastconv.py index 038e2fc..3acc9ef 100644 --- a/src/triacontagon/deprecated/fastconv.py +++ b/src/triacontagon/deprecated/fastconv.py @@ -9,71 +9,11 @@ import torch from .weights import init_glorot from .normalize import _sparse_coo_tensor import types +from .util import _sparse_diag_cat, + _cat -def _sparse_diag_cat(matrices: List[torch.Tensor]): - if len(matrices) == 0: - raise ValueError('The list of matrices must be non-empty') - if not all(m.is_sparse for m in matrices): - raise ValueError('All matrices must be sparse') - - if not all(len(m.shape) == 2 for m in matrices): - raise ValueError('All matrices must be 2D') - - indices = [] - values = [] - row_offset = 0 - col_offset = 0 - - for m in matrices: - ind = m._indices().clone() - ind[0] += row_offset - ind[1] += col_offset - indices.append(ind) - values.append(m._values()) - row_offset += m.shape[0] - col_offset += m.shape[1] - - indices = torch.cat(indices, dim=1) - values = torch.cat(values) - - return _sparse_coo_tensor(indices, values, size=(row_offset, col_offset)) - - -def _cat(matrices: List[torch.Tensor]): - if len(matrices) == 0: - raise ValueError('Empty list passed to _cat()') - - n = sum(a.is_sparse for a in matrices) - if n != 0 and n != len(matrices): - raise ValueError('All matrices must have the same layout (dense or sparse)') - - if not all(a.shape[1:] == matrices[0].shape[1:] for a in matrices): - raise ValueError('All matrices must have the same dimensions apart from dimension 0') - - if not matrices[0].is_sparse: - return torch.cat(matrices) - - total_rows = sum(a.shape[0] for a in matrices) - indices = [] - values = [] - row_offset = 0 - - for a in matrices: - ind = a._indices().clone() - val = a._values() - ind[0] += row_offset - ind = ind.transpose(0, 1) - indices.append(ind) - values.append(val) - row_offset += a.shape[0] - - indices = torch.cat(indices).transpose(0, 1) - values = torch.cat(values) - - res = _sparse_coo_tensor(indices, values, size=(row_offset, matrices[0].shape[1])) - return res class FastGraphConv(torch.nn.Module): diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py index 60b7647..7f79719 100644 --- a/src/triacontagon/sampling.py +++ b/src/triacontagon/sampling.py @@ -12,6 +12,8 @@ from typing import List, \ Tuple from .data import Data, \ EdgeType +from .cumcount import cumcount +import time def fixed_unigram_candidate_sampler( @@ -69,14 +71,62 @@ def get_edges_and_degrees(adj_mat: torch.Tensor) -> \ return edges_pos, degrees +def get_true_classes(adj_mat: torch.Tensor) -> torch.Tensor: + indices = adj_mat.indices() + row_count = torch.zeros(adj_mat.shape[0], dtype=torch.long) + #print('indices[0]:', indices[0], count[indices[0]]) + row_count = row_count.index_add(0, indices[0], + torch.ones(indices.shape[1], dtype=torch.long)) + #print('count:', count) + max_true_classes = torch.max(row_count).item() + #print('max_true_classes:', max_true_classes) + true_classes = torch.full((adj_mat.shape[0], max_true_classes), + -1, dtype=torch.long) + + + # inv = torch.unique(indices[0], return_inverse=True) + + # indices = indices.copy() + # true_classes[indices[0], 0] = indices[1] + t = time.time() + cc = cumcount(indices[0].cpu().numpy()) + print('cumcount() took:', time.time() - t) + cc = torch.tensor(cc) + t = time.time() + true_classes[indices[0], cc] = indices[1] + print('assignment took:', time.time() - t) + + ''' count = torch.zeros(adj_mat.shape[0], dtype=torch.long) + for i in range(indices.shape[1]): + # print('looping...') + row = indices[0, i] + col = indices[1, i] + #print('row:', row, 'col:', col, 'count[row]:', count[row]) + true_classes[row, count[row]] = col + count[row] += 1 ''' + + t = time.time() + true_classes = torch.repeat_interleave(true_classes, row_count, dim=0) + print('repeat_interleave() took:', time.time() - t) + + return true_classes + + def negative_sample_adj_mat(adj_mat: torch.Tensor) -> torch.Tensor: if not isinstance(adj_mat, torch.Tensor): raise ValueError('adj_mat must be a torch.Tensor, got: %s' % adj_mat.__class__.__name__) edges_pos, degrees = get_edges_and_degrees(adj_mat) + true_classes = get_true_classes(adj_mat) + # true_classes = edges_pos[:, 1].view(-1, 1) + # print('true_classes:', true_classes) + neg_neighbors = fixed_unigram_candidate_sampler( - edges_pos[:, 1].view(-1, 1), degrees, 0.75).to(adj_mat.device) + true_classes, degrees, 0.75).to(adj_mat.device) + + print('neg_neighbors:', neg_neighbors) + edges_neg = torch.cat([ edges_pos[:, 0].view(-1, 1), neg_neighbors.view(-1, 1) ], 1) diff --git a/tests/triacontagon/test_sampling.py b/tests/triacontagon/test_sampling.py index 6d7a155..75f056e 100644 --- a/tests/triacontagon/test_sampling.py +++ b/tests/triacontagon/test_sampling.py @@ -1,8 +1,41 @@ from triacontagon.data import Data -from triacontagon.sampling import negative_sample_adj_mat, \ +from triacontagon.sampling import get_true_classes, \ + negative_sample_adj_mat, \ negative_sample_data from triacontagon.decode import dedicom_decoder import torch +import time + + +def test_get_true_classes_01(): + adj_mat = torch.tensor([ + [0, 1, 0, 1, 0], + [0, 0, 0, 0, 1], + [1, 1, 0, 0, 0], + [0, 0, 1, 0, 1], + [0, 1, 0, 0, 0] + ], dtype=torch.float).to_sparse() + + true_classes = get_true_classes(adj_mat) + print('true_classes:', true_classes) + + assert torch.all(true_classes == torch.tensor([ + [1, 3], + [4, -1], + [0, 1], + [2, 4], + [1, -1] + ])) + + +def test_get_true_classes_02(): + adj_mat = torch.rand(2000, 2000).round().to_sparse() + + t = time.time() + true_classes = get_true_classes(adj_mat) + print('Elapsed:', time.time() - t) + + print('true_classes.shape:', true_classes.shape) def test_negative_sample_adj_mat_01(): @@ -16,7 +49,7 @@ def test_negative_sample_adj_mat_01(): print('adj_mat:', adj_mat) - adj_mat_neg = negative_sample_adj_mat(adj_mat) + adj_mat_neg = negative_sample_adj_mat(adj_mat.to_sparse()) print('adj_mat_neg:', adj_mat_neg.to_dense()) -- 2.26.2 From bf39e7d71f25474692ae193064dfa10627e8e81a Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 7 Aug 2020 16:35:18 +0200 Subject: [PATCH 187/227] Small fix. --- tests/triacontagon/test_sampling.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/triacontagon/test_sampling.py b/tests/triacontagon/test_sampling.py index 75f056e..454d3c1 100644 --- a/tests/triacontagon/test_sampling.py +++ b/tests/triacontagon/test_sampling.py @@ -20,16 +20,19 @@ def test_get_true_classes_01(): print('true_classes:', true_classes) assert torch.all(true_classes == torch.tensor([ + [1, 3], [1, 3], [4, -1], [0, 1], + [0, 1], + [2, 4], [2, 4], [1, -1] ])) def test_get_true_classes_02(): - adj_mat = torch.rand(2000, 2000).round().to_sparse() + adj_mat = (torch.rand(2000, 2000) < 0.1).to_sparse() t = time.time() true_classes = get_true_classes(adj_mat) -- 2.26.2 From 56fdd9ffeb2af4d79cee709fcf484a6b5c9eee53 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 7 Aug 2020 18:45:54 +0200 Subject: [PATCH 188/227] Make negative sampling more correct and more efficient at the same time. --- src/triacontagon/sampling.py | 50 ++++++++++++++--------------- tests/triacontagon/test_sampling.py | 8 +++-- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py index 7f79719..58b7ba0 100644 --- a/src/triacontagon/sampling.py +++ b/src/triacontagon/sampling.py @@ -17,41 +17,41 @@ import time def fixed_unigram_candidate_sampler( - true_classes: Union[np.array, torch.Tensor], - unigrams: List[Union[int, float]], + true_classes: torch.Tensor, + num_repeats: torch.Tensor, + unigrams: torch.Tensor, distortion: float = 1.): - if isinstance(true_classes, torch.Tensor): - true_classes = true_classes.detach().cpu().numpy() - - if isinstance(unigrams, torch.Tensor): - unigrams = unigrams.detach().cpu().numpy() - if len(true_classes.shape) != 2: raise ValueError('true_classes must be a 2D matrix with shape (num_samples, num_true)') - num_samples = true_classes.shape[0] - unigrams = np.array(unigrams) + if len(num_repeats.shape) != 1: + raise ValueError('num_repeats must be 1D') + + num_rows = true_classes.shape[0] + # unigrams = np.array(unigrams) if distortion != 1.: - unigrams = unigrams.astype(np.float64) ** distortion + unigrams = unigrams.to(torch.float64) ** distortion # print('unigrams:', unigrams) - indices = np.arange(num_samples) - result = np.zeros(num_samples, dtype=np.int64) + indices = torch.arange(num_rows) + indices = torch.repeat_interleave(indices, num_repeats) + num_samples = len(indices) + result = torch.zeros(num_samples, dtype=torch.long) while len(indices) > 0: # print('len(indices):', len(indices)) sampler = torch.utils.data.WeightedRandomSampler(unigrams, len(indices)) - candidates = np.array(list(sampler)) - candidates = np.reshape(candidates, (len(indices), 1)) + candidates = torch.tensor(list(sampler)) + candidates = candidates.view(len(indices), 1) # print('candidates:', candidates) # print('true_classes:', true_classes[indices, :]) - result[indices] = candidates.T + result[indices] = candidates.transpose(0, 1) # print('result:', result) mask = (candidates == true_classes[indices, :]) - mask = mask.sum(1).astype(np.bool) + mask = mask.sum(1).to(torch.bool) # print('mask:', mask) indices = indices[mask] # result[indices] = 0 - return torch.tensor(result) + return result def get_edges_and_degrees(adj_mat: torch.Tensor) -> \ @@ -71,7 +71,7 @@ def get_edges_and_degrees(adj_mat: torch.Tensor) -> \ return edges_pos, degrees -def get_true_classes(adj_mat: torch.Tensor) -> torch.Tensor: +def get_true_classes(adj_mat: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: indices = adj_mat.indices() row_count = torch.zeros(adj_mat.shape[0], dtype=torch.long) #print('indices[0]:', indices[0], count[indices[0]]) @@ -105,11 +105,11 @@ def get_true_classes(adj_mat: torch.Tensor) -> torch.Tensor: true_classes[row, count[row]] = col count[row] += 1 ''' - t = time.time() - true_classes = torch.repeat_interleave(true_classes, row_count, dim=0) - print('repeat_interleave() took:', time.time() - t) + # t = time.time() + # true_classes = torch.repeat_interleave(true_classes, row_count, dim=0) + # print('repeat_interleave() took:', time.time() - t) - return true_classes + return true_classes, row_count def negative_sample_adj_mat(adj_mat: torch.Tensor) -> torch.Tensor: @@ -118,12 +118,12 @@ def negative_sample_adj_mat(adj_mat: torch.Tensor) -> torch.Tensor: edges_pos, degrees = get_edges_and_degrees(adj_mat) - true_classes = get_true_classes(adj_mat) + true_classes, row_count = get_true_classes(adj_mat) # true_classes = edges_pos[:, 1].view(-1, 1) # print('true_classes:', true_classes) neg_neighbors = fixed_unigram_candidate_sampler( - true_classes, degrees, 0.75).to(adj_mat.device) + true_classes, row_count, degrees, 0.75).to(adj_mat.device) print('neg_neighbors:', neg_neighbors) diff --git a/tests/triacontagon/test_sampling.py b/tests/triacontagon/test_sampling.py index 454d3c1..6bba237 100644 --- a/tests/triacontagon/test_sampling.py +++ b/tests/triacontagon/test_sampling.py @@ -16,9 +16,11 @@ def test_get_true_classes_01(): [0, 1, 0, 0, 0] ], dtype=torch.float).to_sparse() - true_classes = get_true_classes(adj_mat) + true_classes, row_count = get_true_classes(adj_mat) print('true_classes:', true_classes) + true_classes = torch.repeat_interleave(true_classes, row_count, dim=0) + assert torch.all(true_classes == torch.tensor([ [1, 3], [1, 3], @@ -32,10 +34,10 @@ def test_get_true_classes_01(): def test_get_true_classes_02(): - adj_mat = (torch.rand(2000, 2000) < 0.1).to_sparse() + adj_mat = torch.rand(2000, 2000).round().to_sparse() t = time.time() - true_classes = get_true_classes(adj_mat) + true_classes, row_count = get_true_classes(adj_mat) print('Elapsed:', time.time() - t) print('true_classes.shape:', true_classes.shape) -- 2.26.2 From 14e3d14b36e1abea59ea64ed6bedf0eb540f57c9 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 10 Aug 2020 17:55:06 +0200 Subject: [PATCH 189/227] Add Batcher. --- docs/cumcount.svg | 2074 +++++++++++++++++++ src/triacontagon/batch.py | 62 + src/triacontagon/data.py | 3 +- src/triacontagon/sampling.py | 2 +- src/triacontagon/{trainprep.py => split.py} | 0 tests/triacontagon/test_batch.py | 119 ++ 6 files changed, 2258 insertions(+), 2 deletions(-) create mode 100644 docs/cumcount.svg create mode 100644 src/triacontagon/batch.py rename src/triacontagon/{trainprep.py => split.py} (100%) create mode 100644 tests/triacontagon/test_batch.py diff --git a/docs/cumcount.svg b/docs/cumcount.svg new file mode 100644 index 0000000..4da5428 --- /dev/null +++ b/docs/cumcount.svg @@ -0,0 +1,2074 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + 2 + 3 + 3 + 4 + 1 + 1 + 5 + 5 + 2 + 3 + 1 + 4 + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + s=argsort: + i=unargsort: + + + + + + + + + + + + + + + 0 + 1 + 5 + 7 + 8 + 10 + 2 + 3 + 12 + 13 + 6 + 9 + 4 + 11 + b=a[s]: + + + + + + + + + + + + + + + 1 + 1 + 2 + 3 + 3 + 4 + 1 + 1 + 5 + 5 + 2 + 3 + 1 + 4 + dfill(b): + + + + + + + + + + + + + + + 1 + 1 + 1 + 1 + 5 + 7 + 10 + 12 + 0 + ...diff: + 5,2,3,2,2 + ...repeat: + 0,0,0,0,0,5,5,7,7,7,10,10,12,12 + ...where: + arange(n)-dfill(b): + 0,1,2,3,4,0,1,0,1,2,0,1,0,1 + (arange(n)-dfill(b))[i]: + + + + + + + + + + + + + + + 0 + 1 + 0 + 0 + 1 + 0 + 2 + 3 + 0 + 1 + 1 + 2 + 4 + 1 + + + + + + + + + + + + + + + 1 + 1 + 2 + 3 + 3 + 4 + 1 + 1 + 5 + 5 + 2 + 3 + 1 + 4 + in: + in: + + diff --git a/src/triacontagon/batch.py b/src/triacontagon/batch.py new file mode 100644 index 0000000..cfb367e --- /dev/null +++ b/src/triacontagon/batch.py @@ -0,0 +1,62 @@ +from .data import Data +from .model import TrainingBatch +import torch + + +def _shuffle(x: torch.Tensor) -> torch.Tensor: + order = torch.randperm(len(x)) + return x[order] + + +class Batcher(object): + def __init__(self, data: Data, batch_size: int=512, + shuffle: bool=True) -> None: + + if not isinstance(data, Data): + raise TypeError('data must be an instance of Data') + + self.data = data + self.batch_size = int(batch_size) + self.shuffle = bool(shuffle) + + def __iter__(self) -> TrainingBatch: + edge_types = list(self.data.edge_types.values()) + + edge_lists = [ [ adj_mat.indices().transpose(0, 1) \ + for adj_mat in et.adjacency_matrices ] \ + for et in edge_types ] + + if self.shuffle: + edge_lists = [ [ _shuffle(lst) for lst in edge_lst ] \ + for edge_lst in edge_lists ] + + offsets = [ [ 0 ] * len(et.adjacency_matrices) \ + for et in edge_types ] + + while True: + candidates = [ edge_idx for edge_idx, edge_ofs in enumerate(offsets) \ + if len([ rel_idx for rel_idx, rel_ofs in enumerate(edge_ofs) \ + if rel_ofs < len(edge_lists[edge_idx][rel_idx]) ]) > 0 ] + if len(candidates) == 0: + break + + edge_idx = torch.randint(0, len(candidates), (1,)).item() + edge_idx = candidates[edge_idx] + candidates = [ rel_idx \ + for rel_idx, rel_ofs in enumerate(offsets[edge_idx]) \ + if rel_ofs < len(edge_lists[edge_idx][rel_idx]) ] + + rel_idx = torch.randint(0, len(candidates), (1,)).item() + rel_idx = candidates[rel_idx] + + lst = edge_lists[edge_idx][rel_idx] + et = edge_types[edge_idx] + ofs = offsets[edge_idx][rel_idx] + lst = lst[ofs:ofs+self.batch_size] + offsets[edge_idx][rel_idx] += self.batch_size + + b = TrainingBatch(et.vertex_type_row, et.vertex_type_column, + rel_idx, lst, torch.full((len(lst),), self.data.target_value, + dtype=torch.float32)) + + yield b diff --git a/src/triacontagon/data.py b/src/triacontagon/data.py index ba2b7f8..3c13a50 100644 --- a/src/triacontagon/data.py +++ b/src/triacontagon/data.py @@ -39,9 +39,10 @@ class Data(object): vertex_types: List[VertexType] edge_types: List[EdgeType] - def __init__(self) -> None: + def __init__(self, target_value: int = 1) -> None: self.vertex_types = [] self.edge_types = {} + self.target_value = int(target_value) def add_vertex_type(self, name: str, count: int) -> None: name = str(name) diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py index 58b7ba0..c85ee1d 100644 --- a/src/triacontagon/sampling.py +++ b/src/triacontagon/sampling.py @@ -147,7 +147,7 @@ def negative_sample_adj_mat(adj_mat: torch.Tensor) -> torch.Tensor: def negative_sample_data(data: Data) -> Data: new_edge_types = {} - res = Data() + res = Data(target_value=0) for vt in data.vertex_types: res.add_vertex_type(vt.name, vt.count) for key, et in data.edge_types.items(): diff --git a/src/triacontagon/trainprep.py b/src/triacontagon/split.py similarity index 100% rename from src/triacontagon/trainprep.py rename to src/triacontagon/split.py diff --git a/tests/triacontagon/test_batch.py b/tests/triacontagon/test_batch.py new file mode 100644 index 0000000..3717832 --- /dev/null +++ b/tests/triacontagon/test_batch.py @@ -0,0 +1,119 @@ +from triacontagon.batch import Batcher +from triacontagon.data import Data +from triacontagon.decode import dedicom_decoder +import torch + + +def test_batcher_01(): + d = Data() + d.add_vertex_type('Gene', 5) + + d.add_edge_type('Gene-Gene', 0, 0, [ + torch.tensor([ + [0, 1, 0, 1, 0], + [0, 0, 0, 0, 1], + [1, 0, 0, 0, 0], + [0, 0, 1, 0, 0], + [0, 0, 0, 1, 0] + ]).to_sparse() + ], dedicom_decoder) + + b = Batcher(d, batch_size=1) + + visited = set() + for t in b: + print(t) + k = tuple(t.edges[0].tolist()) + visited.add(k) + + assert visited == { (0, 1), (0, 3), + (1, 4), (2, 0), (3, 2), (4, 3) } + + +def test_batcher_02(): + d = Data() + d.add_vertex_type('Gene', 5) + + d.add_edge_type('Gene-Gene', 0, 0, [ + torch.tensor([ + [0, 1, 0, 1, 0], + [0, 0, 0, 0, 1], + [1, 0, 0, 0, 0], + [0, 0, 1, 0, 0], + [0, 0, 0, 1, 0] + ]).to_sparse(), + + torch.tensor([ + [1, 0, 1, 0, 0], + [0, 0, 0, 1, 0], + [0, 0, 0, 0, 1], + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 0] + ]).to_sparse() + ], dedicom_decoder) + + b = Batcher(d, batch_size=1) + + visited = set() + for t in b: + print(t) + k = (t.relation_type_index,) + \ + tuple(t.edges[0].tolist()) + visited.add(k) + + assert visited == { (0, 0, 1), (0, 0, 3), + (0, 1, 4), (0, 2, 0), (0, 3, 2), (0, 4, 3), + (1, 0, 0), (1, 0, 2), (1, 1, 3), (1, 2, 4), + (1, 3, 1), (1, 4, 2) } + + +def test_batcher_03(): + d = Data() + d.add_vertex_type('Gene', 5) + d.add_vertex_type('Drug', 4) + + d.add_edge_type('Gene-Gene', 0, 0, [ + torch.tensor([ + [0, 1, 0, 1, 0], + [0, 0, 0, 0, 1], + [1, 0, 0, 0, 0], + [0, 0, 1, 0, 0], + [0, 0, 0, 1, 0] + ]).to_sparse(), + + torch.tensor([ + [1, 0, 1, 0, 0], + [0, 0, 0, 1, 0], + [0, 0, 0, 0, 1], + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 0] + ]).to_sparse() + ], dedicom_decoder) + + d.add_edge_type('Gene-Drug', 0, 1, [ + torch.tensor([ + [0, 1, 0, 0], + [1, 0, 0, 1], + [0, 1, 0, 0], + [0, 0, 1, 0], + [0, 1, 1, 0] + ]).to_sparse() + ], dedicom_decoder) + + b = Batcher(d, batch_size=1) + + visited = set() + for t in b: + print(t) + k = (t.vertex_type_row, t.vertex_type_column, + t.relation_type_index,) + \ + tuple(t.edges[0].tolist()) + visited.add(k) + + assert visited == { (0, 0, 0, 0, 1), (0, 0, 0, 0, 3), + (0, 0, 0, 1, 4), (0, 0, 0, 2, 0), (0, 0, 0, 3, 2), (0, 0, 0, 4, 3), + (0, 0, 1, 0, 0), (0, 0, 1, 0, 2), (0, 0, 1, 1, 3), (0, 0, 1, 2, 4), + (0, 0, 1, 3, 1), (0, 0, 1, 4, 2), + (0, 1, 0, 0, 1), (0, 1, 0, 1, 0), (0, 1, 0, 1, 3), + (0, 1, 0, 2, 1), (0, 1, 0, 3, 2), (0, 1, 0, 4, 1), + (0, 1, 0, 4, 2) } -- 2.26.2 From 8f86fc083392df1dc144a5b06e033c71d9f2011c Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 10 Aug 2020 17:58:42 +0200 Subject: [PATCH 190/227] More batcher tests. --- tests/triacontagon/test_batch.py | 170 +++++++++++++++++++++++-------- 1 file changed, 125 insertions(+), 45 deletions(-) diff --git a/tests/triacontagon/test_batch.py b/tests/triacontagon/test_batch.py index 3717832..f145380 100644 --- a/tests/triacontagon/test_batch.py +++ b/tests/triacontagon/test_batch.py @@ -68,52 +68,132 @@ def test_batcher_02(): def test_batcher_03(): - d = Data() - d.add_vertex_type('Gene', 5) - d.add_vertex_type('Drug', 4) - - d.add_edge_type('Gene-Gene', 0, 0, [ - torch.tensor([ - [0, 1, 0, 1, 0], - [0, 0, 0, 0, 1], - [1, 0, 0, 0, 0], - [0, 0, 1, 0, 0], - [0, 0, 0, 1, 0] - ]).to_sparse(), - - torch.tensor([ - [1, 0, 1, 0, 0], - [0, 0, 0, 1, 0], - [0, 0, 0, 0, 1], - [0, 1, 0, 0, 0], - [0, 0, 1, 0, 0] - ]).to_sparse() - ], dedicom_decoder) - - d.add_edge_type('Gene-Drug', 0, 1, [ - torch.tensor([ - [0, 1, 0, 0], - [1, 0, 0, 1], - [0, 1, 0, 0], - [0, 0, 1, 0], - [0, 1, 1, 0] - ]).to_sparse() - ], dedicom_decoder) - - b = Batcher(d, batch_size=1) - - visited = set() - for t in b: - print(t) + d = Data() + d.add_vertex_type('Gene', 5) + d.add_vertex_type('Drug', 4) + + d.add_edge_type('Gene-Gene', 0, 0, [ + torch.tensor([ + [0, 1, 0, 1, 0], + [0, 0, 0, 0, 1], + [1, 0, 0, 0, 0], + [0, 0, 1, 0, 0], + [0, 0, 0, 1, 0] + ]).to_sparse(), + + torch.tensor([ + [1, 0, 1, 0, 0], + [0, 0, 0, 1, 0], + [0, 0, 0, 0, 1], + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 0] + ]).to_sparse() + ], dedicom_decoder) + + d.add_edge_type('Gene-Drug', 0, 1, [ + torch.tensor([ + [0, 1, 0, 0], + [1, 0, 0, 1], + [0, 1, 0, 0], + [0, 0, 1, 0], + [0, 1, 1, 0] + ]).to_sparse() + ], dedicom_decoder) + + b = Batcher(d, batch_size=1) + + visited = set() + for t in b: + print(t) + k = (t.vertex_type_row, t.vertex_type_column, + t.relation_type_index,) + \ + tuple(t.edges[0].tolist()) + visited.add(k) + + assert visited == { (0, 0, 0, 0, 1), (0, 0, 0, 0, 3), + (0, 0, 0, 1, 4), (0, 0, 0, 2, 0), (0, 0, 0, 3, 2), (0, 0, 0, 4, 3), + (0, 0, 1, 0, 0), (0, 0, 1, 0, 2), (0, 0, 1, 1, 3), (0, 0, 1, 2, 4), + (0, 0, 1, 3, 1), (0, 0, 1, 4, 2), + (0, 1, 0, 0, 1), (0, 1, 0, 1, 0), (0, 1, 0, 1, 3), + (0, 1, 0, 2, 1), (0, 1, 0, 3, 2), (0, 1, 0, 4, 1), + (0, 1, 0, 4, 2) } + + +def test_batcher_04(): + d = Data() + d.add_vertex_type('Gene', 5) + + d.add_edge_type('Gene-Gene', 0, 0, [ + torch.tensor([ + [0, 1, 0, 1, 0], + [0, 0, 0, 0, 1], + [1, 0, 0, 0, 0], + [0, 0, 1, 0, 0], + [0, 0, 0, 1, 0] + ]).to_sparse() + ], dedicom_decoder) + + b = Batcher(d, batch_size=3) + + visited = set() + for t in b: + print(t) + for e in t.edges: + k = tuple(e.tolist()) + visited.add(k) + + assert visited == { (0, 1), (0, 3), + (1, 4), (2, 0), (3, 2), (4, 3) } + + +def test_batcher_05(): + d = Data() + d.add_vertex_type('Gene', 5) + d.add_vertex_type('Drug', 4) + + d.add_edge_type('Gene-Gene', 0, 0, [ + torch.tensor([ + [0, 1, 0, 1, 0], + [0, 0, 0, 0, 1], + [1, 0, 0, 0, 0], + [0, 0, 1, 0, 0], + [0, 0, 0, 1, 0] + ]).to_sparse(), + + torch.tensor([ + [1, 0, 1, 0, 0], + [0, 0, 0, 1, 0], + [0, 0, 0, 0, 1], + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 0] + ]).to_sparse() + ], dedicom_decoder) + + d.add_edge_type('Gene-Drug', 0, 1, [ + torch.tensor([ + [0, 1, 0, 0], + [1, 0, 0, 1], + [0, 1, 0, 0], + [0, 0, 1, 0], + [0, 1, 1, 0] + ]).to_sparse() + ], dedicom_decoder) + + b = Batcher(d, batch_size=5) + + visited = set() + for t in b: + print(t) + for e in t.edges: k = (t.vertex_type_row, t.vertex_type_column, t.relation_type_index,) + \ - tuple(t.edges[0].tolist()) + tuple(e.tolist()) visited.add(k) - assert visited == { (0, 0, 0, 0, 1), (0, 0, 0, 0, 3), - (0, 0, 0, 1, 4), (0, 0, 0, 2, 0), (0, 0, 0, 3, 2), (0, 0, 0, 4, 3), - (0, 0, 1, 0, 0), (0, 0, 1, 0, 2), (0, 0, 1, 1, 3), (0, 0, 1, 2, 4), - (0, 0, 1, 3, 1), (0, 0, 1, 4, 2), - (0, 1, 0, 0, 1), (0, 1, 0, 1, 0), (0, 1, 0, 1, 3), - (0, 1, 0, 2, 1), (0, 1, 0, 3, 2), (0, 1, 0, 4, 1), - (0, 1, 0, 4, 2) } + assert visited == { (0, 0, 0, 0, 1), (0, 0, 0, 0, 3), + (0, 0, 0, 1, 4), (0, 0, 0, 2, 0), (0, 0, 0, 3, 2), (0, 0, 0, 4, 3), + (0, 0, 1, 0, 0), (0, 0, 1, 0, 2), (0, 0, 1, 1, 3), (0, 0, 1, 2, 4), + (0, 0, 1, 3, 1), (0, 0, 1, 4, 2), + (0, 1, 0, 0, 1), (0, 1, 0, 1, 0), (0, 1, 0, 1, 3), + (0, 1, 0, 2, 1), (0, 1, 0, 3, 2), (0, 1, 0, 4, 1), + (0, 1, 0, 4, 2) } -- 2.26.2 From 0a061af5269622a3f7f8769e6592630fad9f24ae Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 18 Aug 2020 12:25:29 +0200 Subject: [PATCH 191/227] Attempt DualBatcher. --- src/triacontagon/batch.py | 61 +++++++++++++++++++++++++++++++++++++++ src/triacontagon/loop.py | 40 +++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 src/triacontagon/loop.py diff --git a/src/triacontagon/batch.py b/src/triacontagon/batch.py index cfb367e..b93ab36 100644 --- a/src/triacontagon/batch.py +++ b/src/triacontagon/batch.py @@ -1,6 +1,7 @@ from .data import Data from .model import TrainingBatch import torch +from functools import reduce def _shuffle(x: torch.Tensor) -> torch.Tensor: @@ -8,6 +9,66 @@ def _shuffle(x: torch.Tensor) -> torch.Tensor: return x[order] +def _same_data_org(pos_data: Data, neg_data: Data): + if len(pos_data.vertex_types) != len(neg_data.vertex_types): + return False + + test = [ pos_data.vertex_types[i].name == neg_data.vertex_types[i].name \ + and pos_data.vertex_types[i].count == neg_data.vertex_types[i].count \ + for i in range(len(pos_data.vertex_types)) ] + if not all(test): + return False + + if not set(pos_data.edge_types.keys()) == \ + set(neg_data.edge_types.keys()): + + return False + + test = [ pos_data.edge_types[i].name == \ + neg_data.edge_types[i].name \ + and pos_data.edge_types[i].vertex_type_row == \ + neg_data.edge_types[i].vertex_type_row \ + and pos_data.edge_types[i].vertex_type_column == \ + neg_data.edge_types[i].vertex_type_column \ + and len(pos_data.edge_types[i].adjacency_matrices) == \ + len(neg_data.edge_types[i].adjacency_matrices) \ + for i in pos_data.edge_types.keys() ] + if not all(test): + return False + + test = [ [ len(pos_data.edge_types[i].adjacency_matrices[k].values()) == \ + len(neg_data.edge_types[i].adjacency_matrices[k].values()) \ + for k in range(len(pos_data.edge_types[i])) ] \ + for i in pos_data.edge_types.keys() ] + test = reduce(list.__add__, test) + if not all(test): + return False + + return True + + +class DualBatcher(object): + def __init__(self, pos_data: Data, neg_data: Data, + batch_size: int=512, shuffle: bool=True) -> None: + + if not isinstance(pos_data, Data): + raise TypeError('pos_data must be an instance of Data') + + if not isinstance(neg_data, Data): + raise TypeError('neg_data must be an instance of Data') + + if not _same_data_org(pos_data, neg_data): + raise ValueError('pos_data and neg_data must have the same organization') + + self.pos_data = pos_data + self.neg_data = neg_data + self.batch_size = int(batch_size) + self.shuffle = bool(shuffle) + + def __iter__(self): + + + class Batcher(object): def __init__(self, data: Data, batch_size: int=512, shuffle: bool=True) -> None: diff --git a/src/triacontagon/loop.py b/src/triacontagon/loop.py new file mode 100644 index 0000000..18565aa --- /dev/null +++ b/src/triacontagon/loop.py @@ -0,0 +1,40 @@ +from .model import Model +from .batch import Batcher + + +class TrainLoop(object): + def __init__(self, model: Model, + pos_batcher: Batcher, + neg_batcher: Batcher, + max_epochs: int = 50) -> None: + + if not isinstance(model, Model): + raise TypeError('model must be an instance of Model') + + if not isinstance(pos_batcher, Batcher): + raise TypeError('pos_batcher must be an instance of Batcher') + + if not isinstance(neg_batcher, Batcher): + raise TypeError('neg_batcher must be an instance of Batcher') + + self.model = model + self.pos_batcher = pos_batcher + self.neg_batcher = neg_batcher + self.max_epochs = int(num_epochs) + + def run_epoch(self) -> None: + pos_it = iter(self.pos_batcher) + neg_it = iter(self.neg_batcher) + + while True: + try: + pos_batch = next(pos_it) + neg_batch = next(neg_it) + except StopIteration: + break + if len(pos_batch.edges) != len(neg_batch.edges): + raise ValueError('Positive and negative batch should have same length') + + def run(self) -> None: + for epoch in range(self.max_epochs): + self.run_epoch() -- 2.26.2 From 6b15ec8c103e48c5b34c553e9a4abb4117b8c6a8 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 18 Aug 2020 13:59:29 +0200 Subject: [PATCH 192/227] Finish DualBatcher. --- src/triacontagon/batch.py | 72 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/src/triacontagon/batch.py b/src/triacontagon/batch.py index b93ab36..ec66f5f 100644 --- a/src/triacontagon/batch.py +++ b/src/triacontagon/batch.py @@ -21,7 +21,7 @@ def _same_data_org(pos_data: Data, neg_data: Data): if not set(pos_data.edge_types.keys()) == \ set(neg_data.edge_types.keys()): - + return False test = [ pos_data.edge_types[i].name == \ @@ -65,8 +65,78 @@ class DualBatcher(object): self.batch_size = int(batch_size) self.shuffle = bool(shuffle) + def get_edge_lists(self, data: Data): + edge_types = list(data.edge_types.items()) + edge_keys = [ a[0] for a in edge_types ] + edge_types = [ a[1] for a in edge_types ] + + edge_lists = [ [ adj_mat.indices().transpose(0, 1) \ + for adj_mat in et.adjacency_matrices ] \ + for et in edge_types ] + + if self.shuffle: + edge_lists = [ [ _shuffle(lst) for lst in edge_lst ] \ + for edge_lst in edge_lists ] + + offsets = [ [ 0 ] * len(et.adjacency_matrices) \ + for et in edge_types ] + + return (edge_keys, edge_types, edge_lists, offsets) + + def get_candidates(self, edge_lists, offsets): + candidates = [ edge_idx for edge_idx, edge_ofs in enumerate(offsets) \ + if len([ rel_idx for rel_idx, rel_ofs in enumerate(edge_ofs) \ + if rel_ofs < len(edge_lists[edge_idx][rel_idx]) ]) > 0 ] + + if len(candidates) == 0: + return None, None + + edge_idx = torch.randint(0, len(candidates), (1,)).item() + edge_idx = candidates[edge_idx] + candidates = [ rel_idx \ + for rel_idx, rel_ofs in enumerate(offsets[edge_idx]) \ + if rel_ofs < len(edge_lists[edge_idx][rel_idx]) ] + + rel_idx = torch.randint(0, len(candidates), (1,)).item() + rel_idx = candidates[rel_idx] + + return edge_idx, rel_idx + + def take_edges(self, edge_idx, rel_idx, edge_lists, offsets, + edge_types, target_value): + + lst = edge_lists[edge_idx][rel_idx] + et = edge_types[edge_idx] + ofs = offsets[edge_idx][rel_idx] + lst = lst[ofs:ofs+self.batch_size] + offsets[edge_idx][rel_idx] += self.batch_size + + res = TrainingBatch(et.vertex_type_row, et.vertex_type_column, + rel_idx, lst, torch.full(len(lst), target_value, + dtype=torch.float32)) + + return res + def __iter__(self): + pos_edge_keys, pos_edge_types, pos_edge_lists, pos_offsets = \ + self.get_edge_lists(self.pos_data) + + neg_edge_keys, neg_edge_types, neg_edge_lists, neg_offsets = \ + self.get_edge_lists(self.neg_data) + + while True: + edge_idx, rel_idx = self.get_candidates(pos_edge_lists, pos_offsets) + + if edge_idx is None: + return + + pos_batch = self.take_edges(edge_idx, rel_idx, pos_edge_lists, + pos_offsets, pos_edge_types, 1) + + neg_batch = self.take_edges(edge_idx, rel_idx, neg_edge_lists, + neg_offsets, neg_edge_types, 0) + yield (pos_batch, neg_batch) class Batcher(object): -- 2.26.2 From c206df638a115af7fa7e2fff39e16e080025b8b3 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 18 Aug 2020 14:46:23 +0200 Subject: [PATCH 193/227] Add tests for _same_data_org() and DualBatcher. --- src/triacontagon/batch.py | 6 +- tests/triacontagon/test_batch.py | 119 ++++++++++++++++++++++++++++++- 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/src/triacontagon/batch.py b/src/triacontagon/batch.py index ec66f5f..9688d03 100644 --- a/src/triacontagon/batch.py +++ b/src/triacontagon/batch.py @@ -38,9 +38,9 @@ def _same_data_org(pos_data: Data, neg_data: Data): test = [ [ len(pos_data.edge_types[i].adjacency_matrices[k].values()) == \ len(neg_data.edge_types[i].adjacency_matrices[k].values()) \ - for k in range(len(pos_data.edge_types[i])) ] \ + for k in range(len(pos_data.edge_types[i].adjacency_matrices)) ] \ for i in pos_data.edge_types.keys() ] - test = reduce(list.__add__, test) + test = reduce(list.__add__, test, []) if not all(test): return False @@ -112,7 +112,7 @@ class DualBatcher(object): offsets[edge_idx][rel_idx] += self.batch_size res = TrainingBatch(et.vertex_type_row, et.vertex_type_column, - rel_idx, lst, torch.full(len(lst), target_value, + rel_idx, lst, torch.full(( len(lst), ), target_value, dtype=torch.float32)) return res diff --git a/tests/triacontagon/test_batch.py b/tests/triacontagon/test_batch.py index f145380..46ce6a3 100644 --- a/tests/triacontagon/test_batch.py +++ b/tests/triacontagon/test_batch.py @@ -1,9 +1,59 @@ -from triacontagon.batch import Batcher +from triacontagon.batch import _same_data_org, \ + DualBatcher, \ + Batcher from triacontagon.data import Data from triacontagon.decode import dedicom_decoder import torch +def test_same_data_org_01(): + data = Data() + assert _same_data_org(data, data) + + data.add_vertex_type('Foo', 10) + assert _same_data_org(data, data) + + data.add_vertex_type('Bar', 10) + assert _same_data_org(data, data) + + data_1 = Data() + assert not _same_data_org(data, data_1) + + data_1.add_vertex_type('Foo', 10) + assert not _same_data_org(data, data_1) + + data_1.add_vertex_type('Bar', 10) + assert _same_data_org(data, data_1) + + +def test_same_data_org_02(): + data = Data() + data.add_vertex_type('Foo', 4) + data.add_edge_type('Foo-Foo', 0, 0, [ + torch.tensor([ + [0, 0, 0, 1], + [1, 0, 0, 0], + [0, 1, 1, 0], + [1, 0, 1, 0] + ]).to_sparse() + ], dedicom_decoder) + + assert _same_data_org(data, data) + + data_1 = Data() + data_1.add_vertex_type('Foo', 4) + data_1.add_edge_type('Foo-Foo', 0, 0, [ + torch.tensor([ + [0, 0, 0, 1], + [1, 0, 0, 0], + [0, 1, 1, 0], + [1, 0, 0, 0] + ]).to_sparse() + ], dedicom_decoder) + + assert not _same_data_org(data, data_1) + + def test_batcher_01(): d = Data() d.add_vertex_type('Gene', 5) @@ -197,3 +247,70 @@ def test_batcher_05(): (0, 1, 0, 0, 1), (0, 1, 0, 1, 0), (0, 1, 0, 1, 3), (0, 1, 0, 2, 1), (0, 1, 0, 3, 2), (0, 1, 0, 4, 1), (0, 1, 0, 4, 2) } + + +def test_dual_batcher_01(): + d = Data() + d.add_vertex_type('Gene', 5) + d.add_vertex_type('Drug', 4) + + d.add_edge_type('Gene-Gene', 0, 0, [ + torch.tensor([ + [0, 1, 0, 1, 0], + [0, 0, 0, 0, 1], + [1, 0, 0, 0, 0], + [0, 0, 1, 0, 0], + [0, 0, 0, 1, 0] + ]).to_sparse(), + + torch.tensor([ + [1, 0, 1, 0, 0], + [0, 0, 0, 1, 0], + [0, 0, 0, 0, 1], + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 0] + ]).to_sparse() + ], dedicom_decoder) + + d.add_edge_type('Gene-Drug', 0, 1, [ + torch.tensor([ + [0, 1, 0, 0], + [1, 0, 0, 1], + [0, 1, 0, 0], + [0, 0, 1, 0], + [0, 1, 1, 0] + ]).to_sparse() + ], dedicom_decoder) + + b = DualBatcher(d, d, batch_size=5) + + visited_pos = set() + visited_neg = set() + for t_pos, t_neg in b: + assert t_pos.vertex_type_row == t_neg.vertex_type_row + assert t_pos.vertex_type_column == t_neg.vertex_type_column + assert t_pos.relation_type_index == t_neg.relation_type_index + assert len(t_pos.edges) == len(t_neg.edges) + + for e in t_pos.edges: + k = (t_pos.vertex_type_row, t_pos.vertex_type_column, + t_pos.relation_type_index,) + \ + tuple(e.tolist()) + visited_pos.add(k) + + for e in t_neg.edges: + k = (t_neg.vertex_type_row, t_neg.vertex_type_column, + t_neg.relation_type_index,) + \ + tuple(e.tolist()) + visited_neg.add(k) + + expected = { (0, 0, 0, 0, 1), (0, 0, 0, 0, 3), + (0, 0, 0, 1, 4), (0, 0, 0, 2, 0), (0, 0, 0, 3, 2), (0, 0, 0, 4, 3), + (0, 0, 1, 0, 0), (0, 0, 1, 0, 2), (0, 0, 1, 1, 3), (0, 0, 1, 2, 4), + (0, 0, 1, 3, 1), (0, 0, 1, 4, 2), + (0, 1, 0, 0, 1), (0, 1, 0, 1, 0), (0, 1, 0, 1, 3), + (0, 1, 0, 2, 1), (0, 1, 0, 3, 2), (0, 1, 0, 4, 1), + (0, 1, 0, 4, 2) } + + assert visited_pos == expected + assert visited_neg == expected -- 2.26.2 From 4c8c06c63c13fbdb8ac88306b86e2386d542d1ae Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 18 Aug 2020 16:53:00 +0200 Subject: [PATCH 194/227] Add triacontagon.TrainLoop. --- src/triacontagon/loop.py | 115 +++++++++++++++++++++++++++++--------- src/triacontagon/model.py | 11 ++-- 2 files changed, 96 insertions(+), 30 deletions(-) diff --git a/src/triacontagon/loop.py b/src/triacontagon/loop.py index 18565aa..cc20b5a 100644 --- a/src/triacontagon/loop.py +++ b/src/triacontagon/loop.py @@ -1,40 +1,105 @@ from .model import Model from .batch import Batcher +from .sampling import negative_sample_data +from .data import Data -class TrainLoop(object): - def __init__(self, model: Model, - pos_batcher: Batcher, - neg_batcher: Batcher, - max_epochs: int = 50) -> None: +def _merge_pos_neg_batches(pos_batch, neg_batch): + assert len(pos_batch.edges) == len(neg_batch.edges) + assert pos_batch.vertex_type_row == neg_batch.vertex_type_row + assert pos_batch.vertex_type_column == neg_batch.vertex_type_column + assert pos_batch.relation_type_index == neg_batch.relation_type_index - if not isinstance(model, Model): - raise TypeError('model must be an instance of Model') + batch = TrainingBatch(pos_batch.vertex_type_row, + pos_batch.vertex_type_column, + pos_batch.relation_type_index, + torch.cat([ pos_batch.edges, neg_batch.edges ]), + torch.cat([ + torch.ones(len(pos_batch.edges)), + torch.zeros(len(neg_batch.edges)) + ])) + return batch - if not isinstance(pos_batcher, Batcher): - raise TypeError('pos_batcher must be an instance of Batcher') - if not isinstance(neg_batcher, Batcher): - raise TypeError('neg_batcher must be an instance of Batcher') +class TrainLoop(object): + def __init__(self, model: Model, + val_data: Data, test_data: Data, + initial_repr: List[torch.Tensor], + max_epochs: int = 50, + batch_size: int = 512, + loss: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = \ + torch.nn.functional.binary_cross_entropy_with_logits, + lr: float = 0.001) -> None: + + assert isinstance(model, Model) + assert isinstance(val_data, Data) + assert isinstance(test_data, Data) + assert callable(loss) self.model = model - self.pos_batcher = pos_batcher - self.neg_batcher = neg_batcher + self.test_data = test_data + self.initial_repr = list(initial_repr) self.max_epochs = int(num_epochs) + self.batch_size = int(batch_size) + self.loss = loss + self.lr = float(lr) + + self.pos_data = model.data + self.neg_data = negative_sample_data(model.data) + + self.pos_val_data = val_data + self.neg_val_data = negative_sample_data(val_data) + + self.batcher = DualBatcher(self.pos_data, self.neg_data, + batch_size=batch_size) + self.val_batcher = DualBatcher(self.pos_val_data, self.neg_val_data) + + self.opt = torch.optim.Adam(self.model.parameters(), lr=self.lr) def run_epoch(self) -> None: - pos_it = iter(self.pos_batcher) - neg_it = iter(self.neg_batcher) - - while True: - try: - pos_batch = next(pos_it) - neg_batch = next(neg_it) - except StopIteration: - break - if len(pos_batch.edges) != len(neg_batch.edges): - raise ValueError('Positive and negative batch should have same length') + loss_sum = 0. + for pos_batch, neg_batch in self.batcher: + batch = _merge_pos_neg_batches(pos_batch, neg_batch) + + self.opt.zero_grad() + last_layer_repr = self.model.convolve(self.initial_repr) + pred = self.model.decode(last_layer_repr, batch) + loss = self.loss(pred, batch.target_values) + loss.backward() + self.opt.step() + + loss = loss.detach().cpu().item() + loss_sum += loss + print('loss:', loss) + return loss_sum + + def validate_epoch(self): + loss_sum = 0. + for pos_batch, neg_batch in self.val_batcher: + batch = _merge_pos_neg_batches(pos_batch, neg_batch) + with torch.no_grad(): + last_layer_repr = self.model.convolve(self.initial_repr, eval_mode=True) + pred = self.model.decode(last_layer_repr, batch, eval_mode=True) + loss = self.loss(pred, batch.target_values) + loss = loss.detach().cpu().item() + loss_sum += loss + return loss_sum def run(self) -> None: + best_loss = float('inf') + epochs_without_improvement = 0 for epoch in range(self.max_epochs): - self.run_epoch() + print('Epoch', epoch) + loss_sum = self.run_epoch() + print('train loss_sum:', loss_sum) + loss_sum = self.validate_epoch() + print('val loss_sum:', loss_sum) + if loss_sum >= best_loss: + epochs_without_improvement += 1 + else: + epochs_without_improvement = 0 + best_loss = loss_sum + if epochs_without_improvement == 2: + print('Early stopping after epoch', epoch, 'due to no improvement') + + return (epoch, best_loss, loss_sum) diff --git a/src/triacontagon/model.py b/src/triacontagon/model.py index 1a41b4d..09e6de7 100644 --- a/src/triacontagon/model.py +++ b/src/triacontagon/model.py @@ -130,7 +130,7 @@ class Model(torch.nn.Module): torch.nn.Parameter(local_variation[i]) - def convolve(self, in_layer_repr: List[torch.Tensor]) -> \ + def convolve(self, in_layer_repr: List[torch.Tensor], eval_mode=False) -> \ List[torch.Tensor]: cur_layer_repr = in_layer_repr @@ -145,7 +145,7 @@ class Model(torch.nn.Module): num_relation_types = len(et.adjacency_matrices) x = cur_layer_repr[vt_col] - if self.keep_prob != 1: + if self.keep_prob != 1 and not eval_mode: x = dropout(x, self.keep_prob) # print('a, Layer:', i, 'x.shape:', x.shape) @@ -176,7 +176,7 @@ class Model(torch.nn.Module): return cur_layer_repr def decode(self, last_layer_repr: List[torch.Tensor], - batch: TrainingBatch) -> torch.Tensor: + batch: TrainingBatch, eval_mode=False) -> torch.Tensor: vt_row = batch.vertex_type_row vt_col = batch.vertex_type_column @@ -195,8 +195,9 @@ class Model(torch.nn.Module): in_row = in_row[batch.edges[:, 0]] in_col = in_col[batch.edges[:, 1]] - in_row = dropout(in_row, self.keep_prob) - in_col = dropout(in_col, self.keep_prob) + if self.keep_prob != 1 and not eval_mode: + in_row = dropout(in_row, self.keep_prob) + in_col = dropout(in_col, self.keep_prob) # in_row = in_row.to_dense() # in_col = in_col.to_dense() -- 2.26.2 From ebcd38c91020e26421b7586bea04622c695c2111 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 18 Aug 2020 17:44:57 +0200 Subject: [PATCH 195/227] Add first triacontagon experiment. --- .../triacontagon_run/triacontagon_run.py | 146 ++++++++++++++++++ src/triacontagon/util.py | 3 +- 2 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 experiments/triacontagon_run/triacontagon_run.py diff --git a/experiments/triacontagon_run/triacontagon_run.py b/experiments/triacontagon_run/triacontagon_run.py new file mode 100644 index 0000000..cfa81f0 --- /dev/null +++ b/experiments/triacontagon_run/triacontagon_run.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 + +from triacontagon.data import Data +from triacontagon.split import split_data +from triacontagon.model import Model +from triacontagon.loop import TrainLoop +from triacontagon.decode import dedicom_decoder +from triacontagon.util import common_one_hot_encoding + +import os +import pandas as pd +from bisect import bisect_left +import torch +import sys + + +def index(a, x): + i = bisect_left(a, x) + if i != len(a) and a[i] == x: + return i + raise ValueError + + +def load_data(dev): + path = '/pstore/data/data_science/ref/decagon' + + df_combo = pd.read_csv(os.path.join(path, 'bio-decagon-combo.csv')) + df_effcat = pd.read_csv(os.path.join(path, 'bio-decagon-effectcategories.csv')) + df_mono = pd.read_csv(os.path.join(path, 'bio-decagon-mono.csv')) + df_ppi = pd.read_csv(os.path.join(path, 'bio-decagon-ppi.csv')) + df_tgtall = pd.read_csv(os.path.join(path, 'bio-decagon-targets-all.csv')) + df_tgt = pd.read_csv(os.path.join(path, 'bio-decagon-targets.csv')) + + lst = [ 'df_combo', 'df_effcat', 'df_mono', 'df_ppi', 'df_tgtall', 'df_tgt' ] + + for nam in lst: + print(f'len({nam}): {len(locals()[nam])}') + print(f'{nam}.columns: {locals()[nam].columns}') + + genes = set() + genes = genes.union(df_ppi['Gene 1']).union(df_ppi['Gene 2']) \ + .union(df_tgtall['Gene']).union(df_tgt['Gene']) + genes = sorted(genes) + print('len(genes):', len(genes)) + + drugs = set() + drugs = drugs.union(df_combo['STITCH 1']).union(df_combo['STITCH 2']) \ + .union(df_mono['STITCH']).union(df_tgtall['STITCH']).union(df_tgt['STITCH']) + drugs = sorted(drugs) + print('len(drugs):', len(drugs)) + + data = Data() + data.add_vertex_type('Gene', len(genes)) + data.add_vertex_type('Drug', len(drugs)) + + print('Preparing PPI...') + print('Indexing rows...') + rows = [index(genes, g) for g in df_ppi['Gene 1']] + print('Indexing cols...') + cols = [index(genes, g) for g in df_ppi['Gene 2']] + indices = list(zip(rows, cols)) + indices = torch.tensor(indices).transpose(0, 1) + values = torch.ones(len(rows)) + print('indices.shape:', indices.shape, 'values.shape:', values.shape) + adj_mat = torch.sparse_coo_tensor(indices, values, size=(len(genes),) * 2, + device=dev) + adj_mat = (adj_mat + adj_mat.transpose(0, 1)) / 2 + print('adj_mat created') + data.add_edge_type('PPI', 0, 0, [ adj_mat ], dedicom_decoder) + print('OK') + + print('Preparing Drug-Gene (Target) edges...') + rows = [index(drugs, d) for d in df_tgtall['STITCH']] + cols = [index(genes, g) for g in df_tgtall['Gene']] + indices = list(zip(rows, cols)) + indices = torch.tensor(indices).transpose(0, 1) + values = torch.ones(len(rows)) + adj_mat = torch.sparse_coo_tensor(indices, values, size=(len(drugs), len(genes)), + device=dev) + data.add_edge_type('Drug-Gene', 1, 0, [ adj_mat ], dedicom_decoder) + data.add_edge_type('Gene-Drug', 0, 1, [ adj_mat.transpose(0, 1) ], dedicom_decoder) + print('OK') + + print('Preparing Drug-Drug (Side Effect) edges...') + fam = data.add_relation_family('Drug-Drug (Side Effect)', 1, 1, True) + print('# of side effects:', len(df_combo), 'unique:', len(df_combo['Polypharmacy Side Effect'].unique())) + adjacency_matrices = [] + side_effect_names = [] + for eff, df in df_combo.groupby('Polypharmacy Side Effect'): + sys.stdout.write('.') # print(eff, '...') + sys.stdout.flush() + rows = [index(drugs, d) for d in df['STITCH 1']] + cols = [index(drugs, d) for d in df['STITCH 2']] + indices = list(zip(rows, cols)) + indices = torch.tensor(indices).transpose(0, 1) + values = torch.ones(len(rows)) + adj_mat = torch.sparse_coo_tensor(indices, values, size=(len(drugs), len(drugs)), + device=dev) + adj_mat = (adj_mat + adj_mat.transpose(0, 1)) / 2 + adjacency_matrices.append(adj_mat) + side_effect_names.append(df['Polypharmacy Side Effect']) + fam.add_edge_type('Drug-Drug', 1, 1, adjacency_matrices, dedicom_decoder) + print() + print('OK') + + return data + + +def _wrap(obj, method_name): + orig_fn = getattr(obj, method_name) + def fn(*args, **kwargs): + print(f'{method_name}() :: ENTER') + res = orig_fn(*args, **kwargs) + print(f'{method_name}() :: EXIT') + return res + setattr(obj, method_name, fn) + + +def main(): + dev = torch.device('cuda:0') + data = load_data(dev) + + train_data, val_data, test_data = split_data(data, (.8, .1, .1)) + + n = sum(vt.count for vt in data.vertex_types) + + model = Model(data, [n, 32, 64], keep_prob=.9, + conv_activation=torch.sigmoid, + dec_activation=torch.sigmoid).to(dev) + + initial_repr = common_one_hot_encoding([ vt.count \ + for vt in data.vertex_types ], device=dev) + + loop = TrainLoop(model, val_data, test_data, + initial_repr, max_epochs=50, batch_size=512, + loss=torch.nn.functional.binary_cross_entropy_with_logits, + lr=0.001) + + loop.run() + + with open('/pstore/data/data_science/year/2020/adaszews/models/triacontagon/basic_run.pth', 'wb') as f: + torch.save(model.state_dict(), f) + + +if __name__ == '__main__': + main() diff --git a/src/triacontagon/util.py b/src/triacontagon/util.py index f268f44..70067f8 100644 --- a/src/triacontagon/util.py +++ b/src/triacontagon/util.py @@ -225,7 +225,7 @@ def _select_rows(a: torch.Tensor, rows: torch.Tensor): return res -def common_one_hot_encoding(vertex_type_counts: List[int]) -> \ +def common_one_hot_encoding(vertex_type_counts: List[int], device=None) -> \ List[torch.Tensor]: tot = sum(vertex_type_counts) @@ -241,6 +241,7 @@ def common_one_hot_encoding(vertex_type_counts: List[int]) -> \ ]) val = torch.ones(cnt) x = _sparse_coo_tensor(ind, val, size=(cnt, tot)) + x = x.to(device) res.append(x) ofs += cnt -- 2.26.2 From cee0c23bd76294ff88795a655d56e2564ceaf9d3 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Wed, 19 Aug 2020 12:56:30 +0200 Subject: [PATCH 196/227] Add test_cumcount. --- tests/triacontagon/test_cumcount.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tests/triacontagon/test_cumcount.py diff --git a/tests/triacontagon/test_cumcount.py b/tests/triacontagon/test_cumcount.py new file mode 100644 index 0000000..b9a8780 --- /dev/null +++ b/tests/triacontagon/test_cumcount.py @@ -0,0 +1,26 @@ +from triacontagon.cumcount import dfill, \ + argunsort, \ + cumcount +import numpy as np + + +def test_dfill_01(): + input = np.array([1, 1, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5]) + output = dfill(input) + expected = np.array([0, 0, 0, 0, 0, 5, 5, 7, 7, 7, 10, 10, 12, 12]) + assert np.all(output == expected) + + +def test_argunsort_01(): + input = np.array([1, 1, 2, 3, 3, 4, 1, 1, 5, 5, 2, 3, 1, 4]) + output = np.argsort(input, kind='mergesort') + output = argunsort(output) + expected = np.array([0, 1, 5, 7, 8, 10, 2, 3, 12, 13, 6, 9, 4, 11]) + assert np.all(output == expected) + + +def test_cumcount_01(): + input = np.array([1, 1, 2, 3, 3, 4, 1, 1, 5, 5, 2, 3, 1, 4]) + output = cumcount(input) + expected = np.array([0, 1, 0, 0, 1, 0, 2, 3, 0, 1, 1, 2, 4, 1]) + assert np.all(output == expected) -- 2.26.2 From 2260f55399a55fd1cacdbbe7f10971f1b0568b83 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 20 Aug 2020 10:30:59 +0200 Subject: [PATCH 197/227] Add test_merge_pos_neg_batch_01/02(). --- src/triacontagon/loop.py | 6 ++- tests/triacontagon/test_loop.py | 66 +++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 tests/triacontagon/test_loop.py diff --git a/src/triacontagon/loop.py b/src/triacontagon/loop.py index cc20b5a..f52af87 100644 --- a/src/triacontagon/loop.py +++ b/src/triacontagon/loop.py @@ -1,7 +1,11 @@ -from .model import Model +from .model import Model, \ + TrainingBatch from .batch import Batcher from .sampling import negative_sample_data from .data import Data +import torch +from typing import List, \ + Callable def _merge_pos_neg_batches(pos_batch, neg_batch): diff --git a/tests/triacontagon/test_loop.py b/tests/triacontagon/test_loop.py new file mode 100644 index 0000000..dde1299 --- /dev/null +++ b/tests/triacontagon/test_loop.py @@ -0,0 +1,66 @@ +from triacontagon.loop import _merge_pos_neg_batches +from triacontagon.model import TrainingBatch +import torch +import pytest + + +def test_merge_pos_neg_batches_01(): + b_1 = TrainingBatch(0, 0, 0, torch.tensor([ + [0, 1], + [2, 3], + [4, 5], + [5, 6] + ]), torch.ones(4)) + b_2 = TrainingBatch(0, 0, 0, torch.tensor([ + [1, 6], + [3, 5], + [5, 2], + [4, 1] + ]), torch.zeros(4)) + b = _merge_pos_neg_batches(b_1, b_2) + + assert b.vertex_type_row == 0 + assert b.vertex_type_column == 0 + assert b.relation_type_index == 0 + assert torch.all(b.edges == torch.tensor([ + [0, 1], + [2, 3], + [4, 5], + [5, 6], + [1, 6], + [3, 5], + [5, 2], + [4, 1] + ])) + assert torch.all(b.target_values == \ + torch.cat([ torch.ones(4), torch.zeros(4) ])) + + +def test_merge_pos_neg_batches_02(): + b_1 = TrainingBatch(0, 1, 0, torch.tensor([ + [0, 1], + [2, 3], + [4, 5], + [5, 6] + ]), torch.ones(4)) + b_2 = TrainingBatch(0, 0, 0, torch.tensor([ + [1, 6], + [3, 5], + [5, 2], + [4, 1] + ]), torch.zeros(4)) + print(b_1) + with pytest.raises(AssertionError): + _ = _merge_pos_neg_batches(b_1, b_2) + + b_1.vertex_type_row, b_1.vertex_type_column = \ + b_1.vertex_type_column, b_1.vertex_type_row + print(b_1) + with pytest.raises(AssertionError): + _ = _merge_pos_neg_batches(b_1, b_2) + + b_1.vertex_type_row, b_1.relation_type_index = \ + b_1.relation_type_index, b_1.vertex_type_row + print(b_1) + with pytest.raises(AssertionError): + _ = _merge_pos_neg_batches(b_1, b_2) -- 2.26.2 From 7fa7b7372c87b831cbf4b2dc9d1ac126d95d6703 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 20 Aug 2020 12:21:32 +0200 Subject: [PATCH 198/227] Work on loop, split and sampling. --- src/triacontagon/loop.py | 2 +- src/triacontagon/sampling.py | 43 ++++++++++++++---- src/triacontagon/split.py | 42 ++++++++++++------ tests/triacontagon/test_loop.py | 69 ++++++++++++++++++++++++++++- tests/triacontagon/test_sampling.py | 18 +++++++- 5 files changed, 147 insertions(+), 27 deletions(-) diff --git a/src/triacontagon/loop.py b/src/triacontagon/loop.py index f52af87..e5f96bf 100644 --- a/src/triacontagon/loop.py +++ b/src/triacontagon/loop.py @@ -43,7 +43,7 @@ class TrainLoop(object): self.model = model self.test_data = test_data self.initial_repr = list(initial_repr) - self.max_epochs = int(num_epochs) + self.max_epochs = int(max_epochs) self.batch_size = int(batch_size) self.loss = loss self.lr = float(lr) diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py index c85ee1d..cc30402 100644 --- a/src/triacontagon/sampling.py +++ b/src/triacontagon/sampling.py @@ -20,7 +20,7 @@ def fixed_unigram_candidate_sampler( true_classes: torch.Tensor, num_repeats: torch.Tensor, unigrams: torch.Tensor, - distortion: float = 1.): + distortion: float = 1.) -> torch.Tensor: if len(true_classes.shape) != 2: raise ValueError('true_classes must be a 2D matrix with shape (num_samples, num_true)') @@ -29,26 +29,34 @@ def fixed_unigram_candidate_sampler( raise ValueError('num_repeats must be 1D') num_rows = true_classes.shape[0] + print('true_classes.shape:', true_classes.shape) # unigrams = np.array(unigrams) if distortion != 1.: unigrams = unigrams.to(torch.float64) ** distortion - # print('unigrams:', unigrams) + print('unigrams:', unigrams) + indices = torch.arange(num_rows) indices = torch.repeat_interleave(indices, num_repeats) + indices = torch.cat([ torch.arange(len(indices)).view(-1, 1), + indices.view(-1, 1) ], dim=1) + num_samples = len(indices) result = torch.zeros(num_samples, dtype=torch.long) + print('num_rows:', num_rows, 'num_samples:', num_samples) + while len(indices) > 0: - # print('len(indices):', len(indices)) + print('len(indices):', len(indices)) + print('indices:', indices) sampler = torch.utils.data.WeightedRandomSampler(unigrams, len(indices)) candidates = torch.tensor(list(sampler)) candidates = candidates.view(len(indices), 1) - # print('candidates:', candidates) - # print('true_classes:', true_classes[indices, :]) - result[indices] = candidates.transpose(0, 1) - # print('result:', result) - mask = (candidates == true_classes[indices, :]) + print('candidates:', candidates) + print('true_classes:', true_classes[indices[:, 1], :]) + result[indices[:, 0]] = candidates.transpose(0, 1) + print('result:', result) + mask = (candidates == true_classes[indices[:, 1], :]) mask = mask.sum(1).to(torch.bool) - # print('mask:', mask) + print('mask:', mask) indices = indices[mask] # result[indices] = 0 return result @@ -164,3 +172,20 @@ def negative_sample_data(data: Data) -> Data: #new_edge_types[key] = new_et #res = Data(data.vertex_types, new_edge_types) return res + + +def merge_data(pos_data: Data, neg_data: Data) -> Data: + assert isinstance(pos_data, Data) + assert isinstance(neg_data, Data) + + res = PosNegData() + + for vt in pos_data.vertex_types: + res.add_vertex_type(vt.name, vt.count) + + for key, pos_et in pos_data.edge_types.items(): + neg_et = neg_data.edge_types[key] + res.add_edge_type(pos_et.name, + pos_et.vertex_type_row, pos_et.vertex_type_column, + pos_et.adjacency_matrices, neg_et.adjacency_matrices, + pos_et.decoder_factory) diff --git a/src/triacontagon/split.py b/src/triacontagon/split.py index dd7a12c..68826f1 100644 --- a/src/triacontagon/split.py +++ b/src/triacontagon/split.py @@ -1,8 +1,9 @@ from .data import Data, \ - TrainingBatch, \ EdgeType -from typing import Tuple +from typing import Tuple, \ + List from .util import _sparse_coo_tensor +import torch def split_adj_mat(adj_mat: torch.Tensor, ratios: List[float]): @@ -17,21 +18,30 @@ def split_adj_mat(adj_mat: torch.Tensor, ratios: List[float]): ofs = 0 res = [] for r in ratios: - cnt = r * len(values) - ind = indices[:, ofs:ofs+cnt] - val = values[ofs:ofs+cnt] + # cnt = r * len(values) + + beg = int(ofs * len(values)) + end = int((ofs + r) * len(values)) + ofs += r + + ind = indices[:, beg:end] + val = values[beg:end] res.append(_sparse_coo_tensor(ind, val, adj_mat.shape)) - ofs += cnt + # ofs += cnt return res def split_edge_type(et: EdgeType, ratios: Tuple[float, float, float]): - res = [ [] for _ in range(len(et.adjacency_matrices)) ] + res = [ split_adj_mat(adj_mat, ratios) \ + for adj_mat in et.adjacency_matrices ] - for adj_mat in et.adjacency_matrices: - for i, new_adj_mat in enumerate(split_adj_mat(adj_mat, ratios)): - res[i].append(new_adj_mat) + res = [ EdgeType(et.name, + et.vertex_type_row, + et.vertex_type_column, + [ a[i] for a in res ], + et.decoder_factory, + None ) for i in range(len(ratios)) ] return res @@ -49,11 +59,15 @@ def split_data(data: Data, res = [ {} for _ in range(len(ratios)) ] - for key, et in data.edge_types: + for key, et in data.edge_types.items(): for i, new_et in enumerate(split_edge_type(et, ratios)): res[i][key] = new_et - res = [ Data(data.vertex_types, new_edge_types) \ - for new_edge_types in res ] + res_1 = [] + for new_edge_types in res: + d = Data() + d.vertex_types = data.vertex_types, + d.edge_types = new_edge_types + res_1.append(d) - return res + return res_1 diff --git a/tests/triacontagon/test_loop.py b/tests/triacontagon/test_loop.py index dde1299..7456ab5 100644 --- a/tests/triacontagon/test_loop.py +++ b/tests/triacontagon/test_loop.py @@ -1,5 +1,11 @@ -from triacontagon.loop import _merge_pos_neg_batches -from triacontagon.model import TrainingBatch +from triacontagon.loop import _merge_pos_neg_batches, \ + TrainLoop +from triacontagon.model import TrainingBatch, \ + Model +from triacontagon.data import Data +from triacontagon.decode import dedicom_decoder +from triacontagon.util import common_one_hot_encoding +from triacontagon.split import split_data import torch import pytest @@ -64,3 +70,62 @@ def test_merge_pos_neg_batches_02(): print(b_1) with pytest.raises(AssertionError): _ = _merge_pos_neg_batches(b_1, b_2) + + +def test_train_loop_01(): + data = Data() + data.add_vertex_type('Foo', 5) + data.add_vertex_type('Bar', 4) + + foo_foo = torch.tensor([ + [0, 0, 0, 1, 0], + [0, 0, 1, 0, 0], + [1, 0, 0, 1, 0], + [0, 0, 1, 0, 1], + [0, 1, 0, 0, 0] + ]) + foo_foo = (foo_foo + foo_foo.transpose(0, 1)) / 2 + + foo_bar = torch.tensor([ + [0, 1, 0, 1], + [0, 0, 0, 1], + [0, 1, 0, 0], + [1, 0, 0, 0], + [0, 0, 1, 1] + ]) + bar_foo = foo_bar.transpose(0, 1) + + bar_bar = torch.tensor([ + [0, 0, 1, 0], + [1, 0, 0, 0], + [0, 1, 0, 1], + [0, 1, 0, 0], + ]) + bar_bar = (bar_bar + bar_bar.transpose(0, 1)) / 2 + + data.add_edge_type('Foo-Foo', 0, 0, [ + foo_foo.to_sparse().coalesce() + ], dedicom_decoder) + data.add_edge_type('Foo-Bar', 0, 1, [ + foo_bar.to_sparse().coalesce() + ], dedicom_decoder) + data.add_edge_type('Bar-Foo', 1, 0, [ + bar_foo.to_sparse().coalesce() + ], dedicom_decoder) + data.add_edge_type('Bar-Bar', 1, 1, [ + bar_bar.to_sparse().coalesce() + ], dedicom_decoder) + + initial_repr = common_one_hot_encoding([5, 4]) + + model = Model(data, [9, 3, 6], + keep_prob=1.0, + conv_activation=torch.sigmoid, + dec_activation=torch.sigmoid) + + train_data, val_data, test_data = split_data(data, (.9, .1, .0) ) + + loop = TrainLoop(model, val_data, test_data, initial_repr, + max_epochs=1, batch_size=1) + + _ = loop.run() diff --git a/tests/triacontagon/test_sampling.py b/tests/triacontagon/test_sampling.py index 6bba237..0b45769 100644 --- a/tests/triacontagon/test_sampling.py +++ b/tests/triacontagon/test_sampling.py @@ -1,5 +1,6 @@ from triacontagon.data import Data -from triacontagon.sampling import get_true_classes, \ +from triacontagon.sampling import fixed_unigram_candidate_sampler, \ + get_true_classes, \ negative_sample_adj_mat, \ negative_sample_data from triacontagon.decode import dedicom_decoder @@ -7,6 +8,21 @@ import torch import time +def test_fixed_unigram_candidate_sampler_01(): + true_classes = torch.tensor([[-1], + [-1], + [ 3], + [ 2], + [-1]]) + num_repeats = torch.tensor([0, 0, 1, 1, 0]) + unigrams = torch.tensor([0., 0., 1., 1., 0.], dtype=torch.float64) + distortion = 0.75 + res = fixed_unigram_candidate_sampler(true_classes, num_repeats, + unigrams, distortion) + print('res:', res) + + + def test_get_true_classes_01(): adj_mat = torch.tensor([ [0, 1, 0, 1, 0], -- 2.26.2 From 7094ae87dbe0c99518389a42e8d4b051ca73a278 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 20 Aug 2020 12:26:06 +0200 Subject: [PATCH 199/227] Fix negative_sample_adj_mat(). --- src/triacontagon/sampling.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py index cc30402..13ef5cf 100644 --- a/src/triacontagon/sampling.py +++ b/src/triacontagon/sampling.py @@ -135,7 +135,10 @@ def negative_sample_adj_mat(adj_mat: torch.Tensor) -> torch.Tensor: print('neg_neighbors:', neg_neighbors) - edges_neg = torch.cat([ edges_pos[:, 0].view(-1, 1), + pos_vertices = torch.repeat_interleave(torch.arange(len(adj_mat)), + row_count) + + edges_neg = torch.cat([ pos_vertices.view(-1, 1), neg_neighbors.view(-1, 1) ], 1) adj_mat_neg = torch.sparse_coo_tensor(indices = edges_neg.transpose(0, 1), -- 2.26.2 From 604b81675eae47ce7e89261f510a66d56946467b Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 20 Aug 2020 12:45:51 +0200 Subject: [PATCH 200/227] Add test_split_adj_mat_(01|02|03)(). --- tests/triacontagon/test_sampling.py | 1 - tests/triacontagon/test_split.py | 41 +++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 tests/triacontagon/test_split.py diff --git a/tests/triacontagon/test_sampling.py b/tests/triacontagon/test_sampling.py index 0b45769..e72f7a8 100644 --- a/tests/triacontagon/test_sampling.py +++ b/tests/triacontagon/test_sampling.py @@ -22,7 +22,6 @@ def test_fixed_unigram_candidate_sampler_01(): print('res:', res) - def test_get_true_classes_01(): adj_mat = torch.tensor([ [0, 1, 0, 1, 0], diff --git a/tests/triacontagon/test_split.py b/tests/triacontagon/test_split.py new file mode 100644 index 0000000..2cfc8e7 --- /dev/null +++ b/tests/triacontagon/test_split.py @@ -0,0 +1,41 @@ +from triacontagon.split import split_adj_mat +from triacontagon.util import _equal +import torch + + +def test_split_adj_mat_01(): + adj_mat = torch.tensor([ + [0, 1, 0, 0, 1], + [0, 0, 1, 0, 1], + [1, 0, 0, 1, 0], + [0, 0, 1, 1, 0] + ]).to_sparse() + + (res,) = split_adj_mat(adj_mat, (1.,)) + assert torch.all(_equal(res, adj_mat)) + + +def test_split_adj_mat_02(): + adj_mat = torch.tensor([ + [0, 1, 0, 0, 1], + [0, 0, 1, 0, 1], + [1, 0, 0, 1, 0], + [0, 0, 1, 1, 0] + ]).to_sparse() + + a, b = split_adj_mat(adj_mat, ( .5, .5 )) + assert torch.all(_equal(a+b, adj_mat)) + + +def test_split_adj_mat_03(): + adj_mat = torch.tensor([ + [0, 1, 0, 0, 1], + [0, 0, 1, 0, 1], + [1, 0, 0, 1, 0], + [0, 0, 1, 1, 0] + ]).to_sparse() + + a, b, c = split_adj_mat(adj_mat, ( .8, .1, .1 )) + print('a:', a.to_dense(), 'b:', b.to_dense(), 'c:', c.to_dense()) + + assert torch.all(_equal(a+b+c, adj_mat)) -- 2.26.2 From 6d27e26fa765180ea6aca324123ca16c27146937 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 20 Aug 2020 13:41:47 +0200 Subject: [PATCH 201/227] Add test_split_edge_type_01/02/03/04(). --- src/triacontagon/split.py | 8 +++ tests/triacontagon/test_split.py | 85 +++++++++++++++++++++++++++++++- 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/src/triacontagon/split.py b/src/triacontagon/split.py index 68826f1..77f5eaf 100644 --- a/src/triacontagon/split.py +++ b/src/triacontagon/split.py @@ -7,6 +7,10 @@ import torch def split_adj_mat(adj_mat: torch.Tensor, ratios: List[float]): + ratios = list(ratios) + if sum(ratios) != 1: + raise ValueError('Sum of ratios must be 1') + indices = adj_mat.indices() values = adj_mat.values() @@ -33,6 +37,10 @@ def split_adj_mat(adj_mat: torch.Tensor, ratios: List[float]): def split_edge_type(et: EdgeType, ratios: Tuple[float, float, float]): + ratios = list(ratios) + if sum(ratios) != 1: + raise ValueError('Sum of ratios must be 1') + res = [ split_adj_mat(adj_mat, ratios) \ for adj_mat in et.adjacency_matrices ] diff --git a/tests/triacontagon/test_split.py b/tests/triacontagon/test_split.py index 2cfc8e7..85ddc9d 100644 --- a/tests/triacontagon/test_split.py +++ b/tests/triacontagon/test_split.py @@ -1,5 +1,7 @@ -from triacontagon.split import split_adj_mat +from triacontagon.split import split_adj_mat, \ + split_edge_type from triacontagon.util import _equal +from triacontagon.data import EdgeType import torch @@ -39,3 +41,84 @@ def test_split_adj_mat_03(): print('a:', a.to_dense(), 'b:', b.to_dense(), 'c:', c.to_dense()) assert torch.all(_equal(a+b+c, adj_mat)) + + +def test_split_edge_type_01(): + et = EdgeType('Dummy', 0, 1, [ + torch.tensor([ + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 1], + [1, 0, 0, 0, 1], + [0, 1, 0, 1, 0] + ]).to_sparse() + ], None, None) + + res = split_edge_type(et, (1.,)) + + assert torch.all(_equal(et.adjacency_matrices[0], + res[0].adjacency_matrices[0])) + + +def test_split_edge_type_02(): + et = EdgeType('Dummy', 0, 1, [ + torch.tensor([ + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 1], + [1, 0, 0, 0, 1], + [0, 1, 0, 1, 0] + ]).to_sparse() + ], None, None) + + res = split_edge_type(et, (.5, .5)) + + assert torch.all(_equal(et.adjacency_matrices[0], + res[0].adjacency_matrices[0] + \ + res[1].adjacency_matrices[0])) + + +def test_split_edge_type_03(): + et = EdgeType('Dummy', 0, 1, [ + torch.tensor([ + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 1], + [1, 0, 0, 0, 1], + [0, 1, 0, 1, 0] + ]).to_sparse() + ], None, None) + + res = split_edge_type(et, (.4, .4, .2)) + + assert torch.all(_equal(et.adjacency_matrices[0], + res[0].adjacency_matrices[0] + \ + res[1].adjacency_matrices[0] + \ + res[2].adjacency_matrices[0])) + + +def test_split_edge_type_04(): + et = EdgeType('Dummy', 0, 1, [ + torch.tensor([ + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 1], + [1, 0, 0, 0, 1], + [0, 1, 0, 1, 0] + ]).to_sparse(), + + torch.tensor([ + [1, 0, 0, 0, 0], + [0, 1, 0, 1, 0], + [0, 0, 1, 1, 0], + [1, 0, 1, 0, 0] + ]).to_sparse() + ], None, None) + + res = split_edge_type(et, (.4, .4, .2)) + + assert torch.all(_equal(et.adjacency_matrices[0], + res[0].adjacency_matrices[0] + \ + res[1].adjacency_matrices[0] + \ + res[2].adjacency_matrices[0])) + + assert torch.all(_equal(et.adjacency_matrices[1], + res[0].adjacency_matrices[1] + \ + res[1].adjacency_matrices[1] + \ + res[2].adjacency_matrices[1])) -- 2.26.2 From e18469936512aba6baa122adf1b9a1c5b8f6d518 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 20 Aug 2020 17:00:51 +0200 Subject: [PATCH 202/227] Add test_split_data_01/02(). --- tests/triacontagon/test_loop.py | 6 +- tests/triacontagon/test_split.py | 129 ++++++++++++++++++++++++++++++- 2 files changed, 130 insertions(+), 5 deletions(-) diff --git a/tests/triacontagon/test_loop.py b/tests/triacontagon/test_loop.py index 7456ab5..dcc975a 100644 --- a/tests/triacontagon/test_loop.py +++ b/tests/triacontagon/test_loop.py @@ -83,7 +83,7 @@ def test_train_loop_01(): [1, 0, 0, 1, 0], [0, 0, 1, 0, 1], [0, 1, 0, 0, 0] - ]) + ], dtype=torch.float32) foo_foo = (foo_foo + foo_foo.transpose(0, 1)) / 2 foo_bar = torch.tensor([ @@ -92,7 +92,7 @@ def test_train_loop_01(): [0, 1, 0, 0], [1, 0, 0, 0], [0, 0, 1, 1] - ]) + ], dtype=torch.float32) bar_foo = foo_bar.transpose(0, 1) bar_bar = torch.tensor([ @@ -100,7 +100,7 @@ def test_train_loop_01(): [1, 0, 0, 0], [0, 1, 0, 1], [0, 1, 0, 0], - ]) + ], dtype=torch.float32) bar_bar = (bar_bar + bar_bar.transpose(0, 1)) / 2 data.add_edge_type('Foo-Foo', 0, 0, [ diff --git a/tests/triacontagon/test_split.py b/tests/triacontagon/test_split.py index 85ddc9d..dc5459b 100644 --- a/tests/triacontagon/test_split.py +++ b/tests/triacontagon/test_split.py @@ -1,7 +1,10 @@ from triacontagon.split import split_adj_mat, \ - split_edge_type + split_edge_type, \ + split_data from triacontagon.util import _equal -from triacontagon.data import EdgeType +from triacontagon.data import EdgeType, \ + Data +from triacontagon.decode import dedicom_decoder import torch @@ -122,3 +125,125 @@ def test_split_edge_type_04(): res[0].adjacency_matrices[1] + \ res[1].adjacency_matrices[1] + \ res[2].adjacency_matrices[1])) + + +def test_split_data_01(): + data = Data() + data.add_vertex_type('Foo', 5) + data.add_vertex_type('Bar', 4) + + foo_foo = torch.tensor([ + [0, 1, 0, 1, 0], + [0, 0, 0, 1, 0], + [0, 1, 0, 0, 1], + [0, 1, 0, 0, 0], + [1, 0, 0, 1, 0] + ], dtype=torch.float32) + foo_foo = (foo_foo + foo_foo.transpose(0, 1)) / 2 + + foo_bar = torch.tensor([ + [0, 1, 0, 1], + [0, 0, 0, 1], + [0, 1, 0, 0], + [1, 0, 0, 0], + [0, 0, 1, 1] + ], dtype=torch.float32) + bar_foo = foo_bar.transpose(0, 1) + + bar_bar = torch.tensor([ + [0, 0, 1, 0], + [1, 0, 0, 0], + [0, 1, 0, 1], + [0, 1, 0, 0], + ], dtype=torch.float32) + bar_bar = (bar_bar + bar_bar.transpose(0, 1)) / 2 + + data.add_edge_type('Foo-Foo', 0, 0, [ + foo_foo.to_sparse().coalesce() + ], dedicom_decoder) + data.add_edge_type('Foo-Bar', 0, 1, [ + foo_bar.to_sparse().coalesce() + ], dedicom_decoder) + data.add_edge_type('Bar-Foo', 1, 0, [ + bar_foo.to_sparse().coalesce() + ], dedicom_decoder) + data.add_edge_type('Bar-Bar', 1, 1, [ + bar_bar.to_sparse().coalesce() + ], dedicom_decoder) + + (res,) = split_data(data, (1.,)) + + assert torch.all(_equal(res.edge_types[0, 0].adjacency_matrices[0], + data.edge_types[0, 0].adjacency_matrices[0])) + + assert torch.all(_equal(res.edge_types[0, 1].adjacency_matrices[0], + data.edge_types[0, 1].adjacency_matrices[0])) + + assert torch.all(_equal(res.edge_types[1, 0].adjacency_matrices[0], + data.edge_types[1, 0].adjacency_matrices[0])) + + assert torch.all(_equal(res.edge_types[1, 1].adjacency_matrices[0], + data.edge_types[1, 1].adjacency_matrices[0])) + + +def test_split_data_02(): + data = Data() + data.add_vertex_type('Foo', 5) + data.add_vertex_type('Bar', 4) + + foo_foo = torch.tensor([ + [0, 1, 0, 1, 0], + [0, 0, 0, 1, 0], + [0, 1, 0, 0, 1], + [0, 1, 0, 0, 0], + [1, 0, 0, 1, 0] + ], dtype=torch.float32) + foo_foo = (foo_foo + foo_foo.transpose(0, 1)) / 2 + + foo_bar = torch.tensor([ + [0, 1, 0, 1], + [0, 0, 0, 1], + [0, 1, 0, 0], + [1, 0, 0, 0], + [0, 0, 1, 1] + ], dtype=torch.float32) + bar_foo = foo_bar.transpose(0, 1) + + bar_bar = torch.tensor([ + [0, 0, 1, 0], + [1, 0, 0, 0], + [0, 1, 0, 1], + [0, 1, 0, 0], + ], dtype=torch.float32) + bar_bar = (bar_bar + bar_bar.transpose(0, 1)) / 2 + + data.add_edge_type('Foo-Foo', 0, 0, [ + foo_foo.to_sparse().coalesce() + ], dedicom_decoder) + data.add_edge_type('Foo-Bar', 0, 1, [ + foo_bar.to_sparse().coalesce() + ], dedicom_decoder) + data.add_edge_type('Bar-Foo', 1, 0, [ + bar_foo.to_sparse().coalesce() + ], dedicom_decoder) + data.add_edge_type('Bar-Bar', 1, 1, [ + bar_bar.to_sparse().coalesce() + ], dedicom_decoder) + + a, b = split_data(data, (.5,.5)) + + assert torch.all(_equal(a.edge_types[0, 0].adjacency_matrices[0] + \ + b.edge_types[0, 0].adjacency_matrices[0], + data.edge_types[0, 0].adjacency_matrices[0])) + + assert torch.all(_equal(a.edge_types[0, 1].adjacency_matrices[0] + \ + b.edge_types[0, 1].adjacency_matrices[0], + data.edge_types[0, 1].adjacency_matrices[0])) + + assert torch.all(_equal(a.edge_types[1, 0].adjacency_matrices[0] + \ + b.edge_types[1, 0].adjacency_matrices[0], + data.edge_types[1, 0].adjacency_matrices[0])) + + assert torch.all(_equal(a.edge_types[1, 1].adjacency_matrices[0] + \ + b.edge_types[1, 1].adjacency_matrices[0], + data.edge_types[1, 1].adjacency_matrices[0])) -- 2.26.2 From 7e83fed37b597d8a4be00a640064f414a81ff592 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 20 Aug 2020 17:30:28 +0200 Subject: [PATCH 203/227] Add test_normalize. --- tests/triacontagon/test_normalize.py | 185 +++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 tests/triacontagon/test_normalize.py diff --git a/tests/triacontagon/test_normalize.py b/tests/triacontagon/test_normalize.py new file mode 100644 index 0000000..e8de28f --- /dev/null +++ b/tests/triacontagon/test_normalize.py @@ -0,0 +1,185 @@ +from triacontagon.normalize import add_eye_sparse, \ + norm_adj_mat_one_node_type_sparse, \ + norm_adj_mat_one_node_type_dense, \ + norm_adj_mat_one_node_type, \ + norm_adj_mat_two_node_types_sparse, \ + norm_adj_mat_two_node_types_dense, \ + norm_adj_mat_two_node_types +import decagon_pytorch.normalize +import torch +import pytest +import numpy as np +from math import sqrt + + +def test_add_eye_sparse_01(): + adj_mat_dense = torch.rand((10, 10)) + adj_mat_sparse = adj_mat_dense.to_sparse() + + adj_mat_dense += torch.eye(10) + adj_mat_sparse = add_eye_sparse(adj_mat_sparse) + + assert torch.all(adj_mat_sparse.to_dense() == adj_mat_dense) + + +def test_add_eye_sparse_02(): + adj_mat_dense = torch.rand((10, 20)) + adj_mat_sparse = adj_mat_dense.to_sparse() + + with pytest.raises(ValueError): + _ = add_eye_sparse(adj_mat_sparse) + + +def test_add_eye_sparse_03(): + adj_mat_dense = torch.rand((10, 10)) + + with pytest.raises(ValueError): + _ = add_eye_sparse(adj_mat_dense) + + +def test_add_eye_sparse_04(): + adj_mat_dense = np.random.rand(10, 10) + + with pytest.raises(ValueError): + _ = add_eye_sparse(adj_mat_dense) + + +def test_norm_adj_mat_one_node_type_sparse_01(): + adj_mat = torch.rand((10, 10)) + adj_mat = (adj_mat > .5).to(torch.float32) + adj_mat = adj_mat.to_sparse() + _ = norm_adj_mat_one_node_type_sparse(adj_mat) + + +def test_norm_adj_mat_one_node_type_sparse_02(): + adj_mat_dense = torch.rand((10, 10)) + adj_mat_dense = (adj_mat_dense > .5).to(torch.float32) + adj_mat_sparse = adj_mat_dense.to_sparse() + adj_mat_sparse = norm_adj_mat_one_node_type_sparse(adj_mat_sparse) + adj_mat_dense = norm_adj_mat_one_node_type_dense(adj_mat_dense) + assert torch.all(adj_mat_sparse.to_dense() - adj_mat_dense < 0.000001) + + +def test_norm_adj_mat_one_node_type_dense_01(): + adj_mat = torch.rand((10, 10)) + adj_mat = (adj_mat > .5) + _ = norm_adj_mat_one_node_type_dense(adj_mat) + + +def test_norm_adj_mat_one_node_type_dense_02(): + adj_mat = torch.tensor([ + [0, 1, 1, 0], # 3 + [1, 0, 1, 0], # 3 + [1, 1, 0, 1], # 4 + [0, 0, 1, 0] # 2 + # 3 3 4 2 + ]) + expect_denom = np.array([ + [ 3, 3, sqrt(3)*2, sqrt(6) ], + [ 3, 3, sqrt(3)*2, sqrt(6) ], + [ sqrt(3)*2, sqrt(3)*2, 4, sqrt(2)*2 ], + [ sqrt(6), sqrt(6), sqrt(2)*2, 2 ] + ], dtype=np.float32) + expect = (adj_mat.detach().cpu().numpy().astype(np.float32) + np.eye(4)) / expect_denom + # expect = np.array([ + # [1/3, 1/3, 1/3, 0], + # [1/3, 1/3, 1/3, 0], + # [1/4, 1/4, 1/4, 1/4], + # [0, 0, 1/2, 1/2] + # ], dtype=np.float32) + res = decagon_pytorch.normalize.norm_adj_mat_one_node_type(adj_mat) + res = res.todense().astype(np.float32) + print('res:', res) + print('expect:', expect) + assert np.all(res - expect < 0.000001) + + +def test_norm_adj_mat_one_node_type_dense_03(): + # adj_mat = torch.rand((10, 10)) + adj_mat = torch.tensor([ + [0, 1, 1, 0, 0], + [1, 0, 1, 0, 1], + [1, 1, 0, .5, .5], + [0, 0, .5, 0, 1], + [0, 1, .5, 1, 0] + ]) + # adj_mat = (adj_mat > .5) + adj_mat_dec = decagon_pytorch.normalize.norm_adj_mat_one_node_type(adj_mat) + adj_mat_ico = norm_adj_mat_one_node_type_dense(adj_mat) + adj_mat_dec = adj_mat_dec.todense() + adj_mat_ico = adj_mat_ico.detach().cpu().numpy() + print('adj_mat_dec:', adj_mat_dec) + print('adj_mat_ico:', adj_mat_ico) + assert np.all(adj_mat_dec - adj_mat_ico < 0.000001) + + +def test_norm_adj_mat_two_node_types_sparse_01(): + adj_mat = torch.rand((10, 20)) + adj_mat = (adj_mat > .5) + adj_mat = adj_mat.to_sparse() + _ = norm_adj_mat_two_node_types_sparse(adj_mat) + + +def test_norm_adj_mat_two_node_types_sparse_02(): + adj_mat_dense = torch.rand((10, 20)) + adj_mat_dense = (adj_mat_dense > .5) + adj_mat_sparse = adj_mat_dense.to_sparse() + adj_mat_sparse = norm_adj_mat_two_node_types_sparse(adj_mat_sparse) + adj_mat_dense = norm_adj_mat_two_node_types_dense(adj_mat_dense) + assert torch.all(adj_mat_sparse.to_dense() - adj_mat_dense < 0.000001) + + +def test_norm_adj_mat_two_node_types_dense_01(): + adj_mat = torch.rand((10, 20)) + adj_mat = (adj_mat > .5) + _ = norm_adj_mat_two_node_types_dense(adj_mat) + + +def test_norm_adj_mat_two_node_types_dense_02(): + adj_mat = torch.tensor([ + [0, 1, 1, 0], # 2 + [1, 0, 1, 0], # 2 + [1, 1, 0, 1], # 3 + [0, 0, 1, 0] # 1 + # 2 2 3 1 + ]) + expect_denom = np.array([ + [ 2, 2, sqrt(6), sqrt(2) ], + [ 2, 2, sqrt(6), sqrt(2) ], + [ sqrt(6), sqrt(6), 3, sqrt(3) ], + [ sqrt(2), sqrt(2), sqrt(3), 1 ] + ], dtype=np.float32) + expect = adj_mat.detach().cpu().numpy().astype(np.float32) / expect_denom + res = decagon_pytorch.normalize.norm_adj_mat_two_node_types(adj_mat) + res = res.todense().astype(np.float32) + print('res:', res) + print('expect:', expect) + assert np.all(res - expect < 0.000001) + + +def test_norm_adj_mat_two_node_types_dense_03(): + adj_mat = torch.tensor([ + [0, 1, 1, 0, 0], + [1, 0, 1, 0, 1], + [1, 1, 0, .5, .5], + [0, 0, .5, 0, 1], + [0, 1, .5, 1, 0] + ]) + adj_mat_dec = decagon_pytorch.normalize.norm_adj_mat_two_node_types(adj_mat) + adj_mat_ico = norm_adj_mat_two_node_types_dense(adj_mat) + adj_mat_dec = adj_mat_dec.todense() + adj_mat_ico = adj_mat_ico.detach().cpu().numpy() + print('adj_mat_dec:', adj_mat_dec) + print('adj_mat_ico:', adj_mat_ico) + assert np.all(adj_mat_dec - adj_mat_ico < 0.000001) + + +def test_norm_adj_mat_two_node_types_dense_04(): + adj_mat = torch.rand((10, 20)) + adj_mat_dec = decagon_pytorch.normalize.norm_adj_mat_two_node_types(adj_mat) + adj_mat_ico = norm_adj_mat_two_node_types_dense(adj_mat) + adj_mat_dec = adj_mat_dec.todense() + adj_mat_ico = adj_mat_ico.detach().cpu().numpy() + print('adj_mat_dec:', adj_mat_dec) + print('adj_mat_ico:', adj_mat_ico) + assert np.all(adj_mat_dec - adj_mat_ico < 0.000001) -- 2.26.2 From 9ff10387282e9303bbbde75cb3f7f41be1341e4d Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 20 Aug 2020 18:30:12 +0200 Subject: [PATCH 204/227] Add extra checks in Data. --- src/triacontagon/data.py | 22 +++++++++++++++++++--- src/triacontagon/sampling.py | 12 ++++++++++-- src/triacontagon/util.py | 23 +++++++++++++++++++++++ tests/triacontagon/test_batch.py | 20 ++++++++++---------- tests/triacontagon/test_model.py | 32 +++++++++++++++++--------------- 5 files changed, 79 insertions(+), 30 deletions(-) diff --git a/src/triacontagon/data.py b/src/triacontagon/data.py index 3c13a50..48b0a58 100644 --- a/src/triacontagon/data.py +++ b/src/triacontagon/data.py @@ -9,7 +9,8 @@ from typing import Callable, \ Tuple, \ List import types -from .util import _nonzero_sum +from .util import _nonzero_sum, \ + _diag import torch @@ -61,13 +62,28 @@ class Data(object): name = str(name) vertex_type_row = int(vertex_type_row) vertex_type_column = int(vertex_type_column) + if not isinstance(adjacency_matrices, list): raise TypeError('adjacency_matrices must be a list of tensors') - if not isinstance(decoder_factory, types.FunctionType): - raise TypeError('decoder_factory must be a function') + + if not callable(decoder_factory): + raise TypeError('decoder_factory must be callable') + if (vertex_type_row, vertex_type_column) in self.edge_types: raise KeyError('Edge type for given combination of row and column already exists') + + if vertex_type_row == vertex_type_column and \ + any(torch.any(_diag(adj_mat).to(torch.bool)) \ + for adj_mat in adjacency_matrices): + raise ValueError('Adjacency matrices for same row/column vertex types must have empty diagonals') + + if any(adj_mat.shape[0] != self.vertex_types[vertex_type_row].count \ + or adj_mat.shape[1] != self.vertex_types[vertex_type_column].count \ + for adj_mat in adjacency_matrices): + raise ValueError('Adjacency matrices must have as many rows as row vertex type count and as many columns as column vertex type count') + total_connectivity = _nonzero_sum(adjacency_matrices) + self.edge_types[vertex_type_row, vertex_type_column] = \ EdgeType(name, vertex_type_row, vertex_type_column, adjacency_matrices, decoder_factory, total_connectivity) diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py index 13ef5cf..29ac224 100644 --- a/src/triacontagon/sampling.py +++ b/src/triacontagon/sampling.py @@ -120,13 +120,18 @@ def get_true_classes(adj_mat: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor] return true_classes, row_count -def negative_sample_adj_mat(adj_mat: torch.Tensor) -> torch.Tensor: +def negative_sample_adj_mat(adj_mat: torch.Tensor, + remove_diagonal: bool=False) -> torch.Tensor: + if not isinstance(adj_mat, torch.Tensor): raise ValueError('adj_mat must be a torch.Tensor, got: %s' % adj_mat.__class__.__name__) edges_pos, degrees = get_edges_and_degrees(adj_mat) true_classes, row_count = get_true_classes(adj_mat) + if remove_diagonal: + true_classes = torch.cat([ torch.arange(len(adj_mat)).view(-1, 1), + true_classes ], dim=1) # true_classes = edges_pos[:, 1].view(-1, 1) # print('true_classes:', true_classes) @@ -164,7 +169,10 @@ def negative_sample_data(data: Data) -> Data: for key, et in data.edge_types.items(): adjacency_matrices_neg = [] for adj_mat in et.adjacency_matrices: - adj_mat_neg = negative_sample_adj_mat(adj_mat) + remove_diagonal = True \ + if et.vertex_type_row == et.vertex_type_column \ + else False + adj_mat_neg = negative_sample_adj_mat(adj_mat, remove_diagonal) adjacency_matrices_neg.append(adj_mat_neg) res.add_edge_type(et.name, et.vertex_type_row, et.vertex_type_column, diff --git a/src/triacontagon/util.py b/src/triacontagon/util.py index 70067f8..27e8524 100644 --- a/src/triacontagon/util.py +++ b/src/triacontagon/util.py @@ -4,6 +4,29 @@ from typing import List, \ import time +def _diag(x: torch.Tensor, make_sparse: bool=False): + if len(x.shape) < 1 or len(x.shape) > 2: + raise ValueError('Matrix or vector expected') + + if not x.is_sparse and not make_sparse: + return torch.diag(x) + + if len(x.shape) == 1: + indices = torch.arange(len(x)).view(1, -1) + indices = torch.cat([ indices, indices ]) + return _sparse_coo_tensor(indices, x.to_dense(), (len(x),) * 2) + + values = x.values() + indices = x.indices() + mask = torch.nonzero(indices[0] == indices[1], as_tuple=True)[0] + indices = torch.flatten(indices[0, mask]) + order = torch.argsort(indices) + values = values[mask][order] + res = torch.zeros(min(x.shape[0], x.shape[1]), dtype=values.dtype) + res[indices] = values + return res + + def _equal(x: torch.Tensor, y: torch.Tensor): if x.is_sparse ^ y.is_sparse: raise ValueError('Cannot mix sparse and dense tensors') diff --git a/tests/triacontagon/test_batch.py b/tests/triacontagon/test_batch.py index 46ce6a3..9591c45 100644 --- a/tests/triacontagon/test_batch.py +++ b/tests/triacontagon/test_batch.py @@ -33,7 +33,7 @@ def test_same_data_org_02(): torch.tensor([ [0, 0, 0, 1], [1, 0, 0, 0], - [0, 1, 1, 0], + [0, 1, 0, 1], [1, 0, 1, 0] ]).to_sparse() ], dedicom_decoder) @@ -46,7 +46,7 @@ def test_same_data_org_02(): torch.tensor([ [0, 0, 0, 1], [1, 0, 0, 0], - [0, 1, 1, 0], + [0, 1, 0, 1], [1, 0, 0, 0] ]).to_sparse() ], dedicom_decoder) @@ -94,7 +94,7 @@ def test_batcher_02(): ]).to_sparse(), torch.tensor([ - [1, 0, 1, 0, 0], + [0, 0, 1, 0, 1], [0, 0, 0, 1, 0], [0, 0, 0, 0, 1], [0, 1, 0, 0, 0], @@ -113,7 +113,7 @@ def test_batcher_02(): assert visited == { (0, 0, 1), (0, 0, 3), (0, 1, 4), (0, 2, 0), (0, 3, 2), (0, 4, 3), - (1, 0, 0), (1, 0, 2), (1, 1, 3), (1, 2, 4), + (1, 0, 2), (1, 0, 4), (1, 1, 3), (1, 2, 4), (1, 3, 1), (1, 4, 2) } @@ -132,7 +132,7 @@ def test_batcher_03(): ]).to_sparse(), torch.tensor([ - [1, 0, 1, 0, 0], + [0, 0, 1, 0, 1], [0, 0, 0, 1, 0], [0, 0, 0, 0, 1], [0, 1, 0, 0, 0], @@ -162,7 +162,7 @@ def test_batcher_03(): assert visited == { (0, 0, 0, 0, 1), (0, 0, 0, 0, 3), (0, 0, 0, 1, 4), (0, 0, 0, 2, 0), (0, 0, 0, 3, 2), (0, 0, 0, 4, 3), - (0, 0, 1, 0, 0), (0, 0, 1, 0, 2), (0, 0, 1, 1, 3), (0, 0, 1, 2, 4), + (0, 0, 1, 0, 2), (0, 0, 1, 0, 4), (0, 0, 1, 1, 3), (0, 0, 1, 2, 4), (0, 0, 1, 3, 1), (0, 0, 1, 4, 2), (0, 1, 0, 0, 1), (0, 1, 0, 1, 0), (0, 1, 0, 1, 3), (0, 1, 0, 2, 1), (0, 1, 0, 3, 2), (0, 1, 0, 4, 1), @@ -211,7 +211,7 @@ def test_batcher_05(): ]).to_sparse(), torch.tensor([ - [1, 0, 1, 0, 0], + [0, 0, 1, 0, 1], [0, 0, 0, 1, 0], [0, 0, 0, 0, 1], [0, 1, 0, 0, 0], @@ -242,7 +242,7 @@ def test_batcher_05(): assert visited == { (0, 0, 0, 0, 1), (0, 0, 0, 0, 3), (0, 0, 0, 1, 4), (0, 0, 0, 2, 0), (0, 0, 0, 3, 2), (0, 0, 0, 4, 3), - (0, 0, 1, 0, 0), (0, 0, 1, 0, 2), (0, 0, 1, 1, 3), (0, 0, 1, 2, 4), + (0, 0, 1, 0, 2), (0, 0, 1, 0, 4), (0, 0, 1, 1, 3), (0, 0, 1, 2, 4), (0, 0, 1, 3, 1), (0, 0, 1, 4, 2), (0, 1, 0, 0, 1), (0, 1, 0, 1, 0), (0, 1, 0, 1, 3), (0, 1, 0, 2, 1), (0, 1, 0, 3, 2), (0, 1, 0, 4, 1), @@ -264,7 +264,7 @@ def test_dual_batcher_01(): ]).to_sparse(), torch.tensor([ - [1, 0, 1, 0, 0], + [0, 0, 1, 0, 1], [0, 0, 0, 1, 0], [0, 0, 0, 0, 1], [0, 1, 0, 0, 0], @@ -306,7 +306,7 @@ def test_dual_batcher_01(): expected = { (0, 0, 0, 0, 1), (0, 0, 0, 0, 3), (0, 0, 0, 1, 4), (0, 0, 0, 2, 0), (0, 0, 0, 3, 2), (0, 0, 0, 4, 3), - (0, 0, 1, 0, 0), (0, 0, 1, 0, 2), (0, 0, 1, 1, 3), (0, 0, 1, 2, 4), + (0, 0, 1, 0, 2), (0, 0, 1, 0, 4), (0, 0, 1, 1, 3), (0, 0, 1, 2, 4), (0, 0, 1, 3, 1), (0, 0, 1, 4, 2), (0, 1, 0, 0, 1), (0, 1, 0, 1, 0), (0, 1, 0, 1, 3), (0, 1, 0, 2, 1), (0, 1, 0, 3, 2), (0, 1, 0, 4, 1), diff --git a/tests/triacontagon/test_model.py b/tests/triacontagon/test_model.py index 0f57578..06c0ece 100644 --- a/tests/triacontagon/test_model.py +++ b/tests/triacontagon/test_model.py @@ -13,10 +13,10 @@ def test_per_layer_required_vertices_01(): d.add_vertex_type('Drug', 5) d.add_edge_type('Gene-Gene', 0, 0, [ torch.tensor([ - [1, 0, 0, 1], - [0, 1, 1, 0], + [0, 0, 0, 1], [0, 0, 1, 0], - [0, 1, 0, 1] + [1, 0, 0, 0], + [0, 1, 0, 0] ]).to_sparse() ], dedicom_decoder) d.add_edge_type('Gene-Drug', 0, 1, [ torch.tensor([ @@ -27,11 +27,11 @@ def test_per_layer_required_vertices_01(): ]).to_sparse() ], dedicom_decoder) d.add_edge_type('Drug-Drug', 1, 1, [ torch.tensor([ + [0, 0, 1, 0, 1], + [0, 0, 0, 1, 1], [1, 0, 0, 0, 0], - [0, 1, 0, 0, 0], - [0, 0, 1, 0, 0], - [0, 0, 0, 1, 0], - [0, 0, 0, 0, 1] + [0, 1, 0, 0, 1], + [1, 1, 0, 1, 0] ]).to_sparse() ], dedicom_decoder) batch = TrainingBatch(0, 1, 0, torch.tensor([ @@ -48,10 +48,10 @@ def test_model_convolve_01(): d.add_vertex_type('Drug', 5) d.add_edge_type('Gene-Gene', 0, 0, [ torch.tensor([ - [1, 0, 0, 1], - [0, 1, 1, 0], + [0, 0, 0, 1], [0, 0, 1, 0], - [0, 1, 0, 1] + [1, 0, 0, 0], + [0, 1, 0, 0] ], dtype=torch.float).to_sparse() ], dedicom_decoder) d.add_edge_type('Gene-Drug', 0, 1, [ torch.tensor([ @@ -62,11 +62,11 @@ def test_model_convolve_01(): ], dtype=torch.float).to_sparse() ], dedicom_decoder) d.add_edge_type('Drug-Drug', 1, 1, [ torch.tensor([ - [1, 0, 0, 0, 0], - [0, 1, 0, 0, 0], - [0, 0, 1, 0, 0], [0, 0, 0, 1, 0], - [0, 0, 0, 0, 1] + [0, 0, 0, 0, 1], + [0, 1, 0, 0, 0], + [1, 0, 0, 0, 0], + [0, 1, 0, 1, 0] ], dtype=torch.float).to_sparse() ], dedicom_decoder) model = Model(d, [9, 32, 64], keep_prob=1.0, @@ -90,8 +90,10 @@ def test_model_decode_01(): d = Data() d.add_vertex_type('Gene', 100) + gene_gene = torch.rand(100, 100).round() + gene_gene = gene_gene - torch.diag(torch.diag(gene_gene)) d.add_edge_type('Gene-Gene', 0, 0, [ - torch.rand(100, 100).round().to_sparse() + gene_gene.to_sparse() ], dedicom_decoder) b = TrainingBatch(0, 0, 0, torch.tensor([ -- 2.26.2 From 9defd6c9e40bf2a7a144d320784437ceba4fd9b3 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 20 Aug 2020 18:43:20 +0200 Subject: [PATCH 205/227] Add test_dropout. --- src/triacontagon/dropout.py | 2 +- tests/triacontagon/test_dropout.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/triacontagon/test_dropout.py diff --git a/src/triacontagon/dropout.py b/src/triacontagon/dropout.py index 2fb8728..e9dde92 100644 --- a/src/triacontagon/dropout.py +++ b/src/triacontagon/dropout.py @@ -26,7 +26,7 @@ def dropout_sparse(x, keep_prob): def dropout_dense(x, keep_prob): # print('dropout_dense()') x = x.clone() - i = torch.nonzero(x) + i = torch.nonzero(x, as_tuple=False) n = keep_prob + torch.rand(len(i)) n = (1. - torch.floor(n)).to(torch.bool) diff --git a/tests/triacontagon/test_dropout.py b/tests/triacontagon/test_dropout.py new file mode 100644 index 0000000..abdb04c --- /dev/null +++ b/tests/triacontagon/test_dropout.py @@ -0,0 +1,26 @@ +from triacontagon.dropout import dropout_sparse, \ + dropout_dense +import torch +import numpy as np + + +def test_dropout_01(): + for i in range(11): + torch.random.manual_seed(i) + a = torch.rand((5, 10)) + a[a < .5] = 0 + + keep_prob=i/10. + np.finfo(np.float32).eps + + torch.random.manual_seed(i) + b = dropout_dense(a, keep_prob=keep_prob) + + torch.random.manual_seed(i) + c = dropout_sparse(a.to_sparse(), keep_prob=keep_prob) + + print('keep_prob:', keep_prob) + print('a:', a.detach().cpu().numpy()) + print('b:', b.detach().cpu().numpy()) + print('c:', c, c.to_dense().detach().cpu().numpy()) + + assert torch.all(b == c.to_dense()) -- 2.26.2 From 8fbad74dfa2fec703345904c76a4a08bdc35a493 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 20 Aug 2020 18:56:52 +0200 Subject: [PATCH 206/227] Debug negative sampling. --- tests/triacontagon/test_sampling.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/triacontagon/test_sampling.py b/tests/triacontagon/test_sampling.py index e72f7a8..088cee3 100644 --- a/tests/triacontagon/test_sampling.py +++ b/tests/triacontagon/test_sampling.py @@ -2,7 +2,8 @@ from triacontagon.data import Data from triacontagon.sampling import fixed_unigram_candidate_sampler, \ get_true_classes, \ negative_sample_adj_mat, \ - negative_sample_data + negative_sample_data, \ + get_edges_and_degrees from triacontagon.decode import dedicom_decoder import torch import time @@ -22,6 +23,29 @@ def test_fixed_unigram_candidate_sampler_01(): print('res:', res) +def test_fixed_unigram_candidate_sampler_02(): + foo_bar = torch.tensor([ + [0, 1, 0, 1], + [0, 0, 0, 1], + [0, 1, 0, 0], + [1, 0, 0, 0], + [0, 0, 1, 1] + ], dtype=torch.float32) + + # bar_foo = foo_bar.transpose(0, 1).to_sparse().coalesce() + bar_foo = foo_bar.to_sparse().coalesce() + + true_classes, row_count = get_true_classes(bar_foo) + print('true_classes:', true_classes) + print('row_count:', row_count) + + edges_pos, degrees = get_edges_and_degrees(bar_foo) + + res = fixed_unigram_candidate_sampler(true_classes, row_count, + degrees, 0.75) + print('res:', res) + + def test_get_true_classes_01(): adj_mat = torch.tensor([ [0, 1, 0, 1, 0], -- 2.26.2 From 2ff358f7efe52c9c193fdab7b952837ee923a2f0 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Thu, 20 Aug 2020 22:06:40 +0200 Subject: [PATCH 207/227] Protect against endless loop. --- src/triacontagon/sampling.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py index 29ac224..8beac99 100644 --- a/src/triacontagon/sampling.py +++ b/src/triacontagon/sampling.py @@ -28,6 +28,11 @@ def fixed_unigram_candidate_sampler( if len(num_repeats.shape) != 1: raise ValueError('num_repeats must be 1D') + if torch.any(len(unigrams) - \ + (true_classes >= 0).sum(dim=1) < \ + num_repeats): + raise ValueError('Not enough classes to choose from') + num_rows = true_classes.shape[0] print('true_classes.shape:', true_classes.shape) # unigrams = np.array(unigrams) -- 2.26.2 From 58b0aac18b5a1c212f981543f4a3baf07d8c0b5f Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 21 Aug 2020 11:07:27 +0200 Subject: [PATCH 208/227] Fixes in sampling. --- src/triacontagon/sampling.py | 4 +++- src/triacontagon/split.py | 6 +++--- tests/triacontagon/test_loop.py | 23 +++++++++++++---------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py index 8beac99..bcec040 100644 --- a/src/triacontagon/sampling.py +++ b/src/triacontagon/sampling.py @@ -28,7 +28,7 @@ def fixed_unigram_candidate_sampler( if len(num_repeats.shape) != 1: raise ValueError('num_repeats must be 1D') - if torch.any(len(unigrams) - \ + if torch.any((unigrams > 0).sum() - \ (true_classes >= 0).sum(dim=1) < \ num_repeats): raise ValueError('Not enough classes to choose from') @@ -132,6 +132,7 @@ def negative_sample_adj_mat(adj_mat: torch.Tensor, raise ValueError('adj_mat must be a torch.Tensor, got: %s' % adj_mat.__class__.__name__) edges_pos, degrees = get_edges_and_degrees(adj_mat) + degrees = degrees.to(torch.float32) + 1.0 / torch.numel(adj_mat) true_classes, row_count = get_true_classes(adj_mat) if remove_diagonal: @@ -172,6 +173,7 @@ def negative_sample_data(data: Data) -> Data: for vt in data.vertex_types: res.add_vertex_type(vt.name, vt.count) for key, et in data.edge_types.items(): + print('key:', key) adjacency_matrices_neg = [] for adj_mat in et.adjacency_matrices: remove_diagonal = True \ diff --git a/src/triacontagon/split.py b/src/triacontagon/split.py index 77f5eaf..c105e4e 100644 --- a/src/triacontagon/split.py +++ b/src/triacontagon/split.py @@ -30,7 +30,7 @@ def split_adj_mat(adj_mat: torch.Tensor, ratios: List[float]): ind = indices[:, beg:end] val = values[beg:end] - res.append(_sparse_coo_tensor(ind, val, adj_mat.shape)) + res.append(_sparse_coo_tensor(ind, val, adj_mat.shape).coalesce()) # ofs += cnt return res @@ -40,7 +40,7 @@ def split_edge_type(et: EdgeType, ratios: Tuple[float, float, float]): ratios = list(ratios) if sum(ratios) != 1: raise ValueError('Sum of ratios must be 1') - + res = [ split_adj_mat(adj_mat, ratios) \ for adj_mat in et.adjacency_matrices ] @@ -74,7 +74,7 @@ def split_data(data: Data, res_1 = [] for new_edge_types in res: d = Data() - d.vertex_types = data.vertex_types, + d.vertex_types = data.vertex_types d.edge_types = new_edge_types res_1.append(d) diff --git a/tests/triacontagon/test_loop.py b/tests/triacontagon/test_loop.py index dcc975a..5f4359a 100644 --- a/tests/triacontagon/test_loop.py +++ b/tests/triacontagon/test_loop.py @@ -78,28 +78,28 @@ def test_train_loop_01(): data.add_vertex_type('Bar', 4) foo_foo = torch.tensor([ - [0, 0, 0, 1, 0], [0, 0, 1, 0, 0], - [1, 0, 0, 1, 0], - [0, 0, 1, 0, 1], - [0, 1, 0, 0, 0] + [0, 0, 0, 1, 0], + [1, 0, 0, 0, 0], + [0, 1, 0, 0, 0], + [0, 0, 0, 0, 0] ], dtype=torch.float32) foo_foo = (foo_foo + foo_foo.transpose(0, 1)) / 2 foo_bar = torch.tensor([ - [0, 1, 0, 1], + [0, 0, 1, 0], [0, 0, 0, 1], [0, 1, 0, 0], [1, 0, 0, 0], - [0, 0, 1, 1] + [0, 0, 0, 1] ], dtype=torch.float32) bar_foo = foo_bar.transpose(0, 1) bar_bar = torch.tensor([ - [0, 0, 1, 0], - [1, 0, 0, 0], - [0, 1, 0, 1], [0, 1, 0, 0], + [1, 0, 0, 0], + [0, 0, 0, 1], + [0, 0, 1, 0], ], dtype=torch.float32) bar_bar = (bar_bar + bar_bar.transpose(0, 1)) / 2 @@ -123,7 +123,10 @@ def test_train_loop_01(): conv_activation=torch.sigmoid, dec_activation=torch.sigmoid) - train_data, val_data, test_data = split_data(data, (.9, .1, .0) ) + train_data, val_data, test_data = split_data(data, (.5, .5, .0) ) + + print('val_data:', val_data) + print('val_data.vertex_types:', val_data.vertex_types) loop = TrainLoop(model, val_data, test_data, initial_repr, max_epochs=1, batch_size=1) -- 2.26.2 From 6cf82515394b8f560853d616ec8fe125c45e8f49 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 21 Aug 2020 12:44:08 +0200 Subject: [PATCH 209/227] fixed_unigram_candidate_sampler() still requires some work to be perfect. --- src/triacontagon/loop.py | 2 +- src/triacontagon/sampling.py | 60 ++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/triacontagon/loop.py b/src/triacontagon/loop.py index e5f96bf..870c07e 100644 --- a/src/triacontagon/loop.py +++ b/src/triacontagon/loop.py @@ -1,6 +1,6 @@ from .model import Model, \ TrainingBatch -from .batch import Batcher +from .batch import DualBatcher from .sampling import negative_sample_data from .data import Data import torch diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py index bcec040..ab033dd 100644 --- a/src/triacontagon/sampling.py +++ b/src/triacontagon/sampling.py @@ -14,6 +14,60 @@ from .data import Data, \ EdgeType from .cumcount import cumcount import time +import multiprocessing +import multiprocessing.pool +from itertools import product, \ + repeat +from functools import reduce + + +def fixed_unigram_candidate_sampler_slow( + true_classes: torch.Tensor, + num_repeats: torch.Tensor, + unigrams: torch.Tensor, + distortion: float = 1.) -> torch.Tensor: + + assert isinstance(true_classes, torch.Tensor) + assert isinstance(num_repeats, torch.Tensor) + assert isinstance(unigrams, torch.Tensor) + distortion = float(distortion) + + if len(true_classes.shape) != 2: + raise ValueError('true_classes must be a 2D matrix with shape (num_samples, num_true)') + + if len(num_repeats.shape) != 1: + raise ValueError('num_repeats must be 1D') + + if torch.any((unigrams > 0).sum() - \ + (true_classes >= 0).sum(dim=1) < \ + num_repeats): + raise ValueError('Not enough classes to choose from') + + res = [] + + if distortion != 1.: + unigrams = unigrams.to(torch.float64) + unigrams = unigrams ** distortion + + def fun(i): + if i and i % 100 == 0: + print(i) + if num_repeats[i] == 0: + return [] + pos = torch.flatten(true_classes[i, :]) + pos = pos[pos >= 0] + w = unigrams.clone().detach() + w[pos] = 0 + sampler = torch.utils.data.WeightedRandomSampler(w, + num_repeats[i].item(), replacement=False) + res = list(sampler) + return res + + with multiprocessing.pool.ThreadPool() as p: + res = p.map(fun, range(len(num_repeats))) + res = reduce(list.__add__, res, []) + + return torch.tensor(res) def fixed_unigram_candidate_sampler( @@ -61,6 +115,12 @@ def fixed_unigram_candidate_sampler( print('result:', result) mask = (candidates == true_classes[indices[:, 1], :]) mask = mask.sum(1).to(torch.bool) + # append_true_classes = torch.full(( len(true_classes), ), -1) + # append_true_classes[~mask] = torch.flatten(candidates)[~mask] + # true_classes = torch.cat([ + # append_true_classes.view(-1, 1), + # true_classes + # ], dim=1) print('mask:', mask) indices = indices[mask] # result[indices] = 0 -- 2.26.2 From 346f99fa83068bf07bc6099f6d3691f5e071ef13 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 21 Aug 2020 17:00:46 +0200 Subject: [PATCH 210/227] Introduce fixed_unigram_candidate_sampler_new(). --- src/triacontagon/cumcount.py | 26 +++++++---- src/triacontagon/sampling.py | 69 ++++++++++++++++++++++++++++- tests/triacontagon/test_cumcount.py | 23 +++++----- tests/triacontagon/test_sampling.py | 12 ++++- 4 files changed, 107 insertions(+), 23 deletions(-) diff --git a/src/triacontagon/cumcount.py b/src/triacontagon/cumcount.py index ba1f23f..faee2be 100644 --- a/src/triacontagon/cumcount.py +++ b/src/triacontagon/cumcount.py @@ -1,22 +1,30 @@ +import torch import numpy as np def dfill(a): - n = a.size - b = np.concatenate([[0], np.where(a[:-1] != a[1:])[0] + 1, [n]]) - return np.arange(n)[b[:-1]].repeat(np.diff(b)) + n = torch.numel(a) + b = torch.cat([ + torch.tensor([0]), + torch.nonzero(a[:-1] != a[1:], as_tuple=True)[0] + 1, + torch.tensor([n]) + ]) + res = torch.arange(n)[b[:-1]] + res = torch.repeat_interleave(res, b[1:] - b[:-1]) + return res def argunsort(s): - n = s.size - u = np.empty(n, dtype=np.int64) - u[s] = np.arange(n) + n = torch.numel(s) + u = torch.empty(n, dtype=torch.int64) + u[s] = torch.arange(n) return u def cumcount(a): - n = a.size - s = a.argsort(kind='mergesort') + n = torch.numel(a) + s = np.argsort(a.detach().cpu().numpy()) + s = torch.tensor(s, device=a.device) i = argunsort(s) b = a[s] - return (np.arange(n) - dfill(b))[i] + return (torch.arange(n) - dfill(b))[i] diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py index ab033dd..76e42dc 100644 --- a/src/triacontagon/sampling.py +++ b/src/triacontagon/sampling.py @@ -21,6 +21,71 @@ from itertools import product, \ from functools import reduce +def fixed_unigram_candidate_sampler_new( + true_classes: torch.Tensor, + num_repeats: torch.Tensor, + unigrams: torch.Tensor, + distortion: float = 1.) -> torch.Tensor: + + assert isinstance(true_classes, torch.Tensor) + assert isinstance(num_repeats, torch.Tensor) + assert isinstance(unigrams, torch.Tensor) + distortion = float(distortion) + + if len(true_classes.shape) != 2: + raise ValueError('true_classes must be a 2D matrix with shape (num_samples, num_true)') + + if len(num_repeats.shape) != 1: + raise ValueError('num_repeats must be 1D') + + if torch.any((unigrams > 0).sum() - \ + (true_classes >= 0).sum(dim=1) < \ + num_repeats): + raise ValueError('Not enough classes to choose from') + + true_class_count = true_classes.shape[1] - (true_classes == -1).sum(dim=1) + true_classes = torch.cat([ + true_classes, + torch.full(( len(true_classes), torch.max(num_repeats) ), -1, + dtype=true_classes.dtype) + ], dim=1) + + indices = torch.repeat_interleave(torch.arange(len(unigrams)), num_repeats) + indices = torch.cat([ torch.arange(len(indices)).view(-1, 1), + indices.view(-1, 1) ], dim=1) + + result = torch.zeros(len(indices), dtype=torch.long) + + while len(indices) > 0: + candidates = torch.utils.data.WeightedRandomSampler(unigrams, len(indices)) + candidates = torch.tensor(list(candidates)).view(-1, 1) + + inner_order = torch.argsort(candidates[:, 0]) + indices_np = indices[inner_order].detach().cpu().numpy() + outer_order = np.argsort(indices_np[:, 1], kind='stable') + outer_order = torch.tensor(outer_order, device=inner_order.device) + + candidates = candidates[inner_order][outer_order] + indices = indices[inner_order][outer_order] + + mask = (true_classes[indices[:, 1]] == candidates).sum(dim=1).to(torch.bool) + + can_cum = cumcount(candidates[:, 0]) + ind_cum = cumcount(indices[:, 1]) + repeated = (can_cum > 0) & (ind_cum > 0) + + mask = mask | repeated + + updated = indices[~mask] + ofs = true_class_count[updated[:, 1]] + \ + cumcount(updated[:, 1]) + true_classes[updated[:, 1], ofs] = candidates[~mask].transpose(0, 1) + true_class_count[updated[:, 1]] = ofs + 1 + + result[indices[:, 0]] = candidates.transpose(0, 1) + indices = indices[mask] + + def fixed_unigram_candidate_sampler_slow( true_classes: torch.Tensor, num_repeats: torch.Tensor, @@ -162,9 +227,9 @@ def get_true_classes(adj_mat: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor] # indices = indices.copy() # true_classes[indices[0], 0] = indices[1] t = time.time() - cc = cumcount(indices[0].cpu().numpy()) + cc = cumcount(indices[0]) print('cumcount() took:', time.time() - t) - cc = torch.tensor(cc) + # cc = torch.tensor(cc) t = time.time() true_classes[indices[0], cc] = indices[1] print('assignment took:', time.time() - t) diff --git a/tests/triacontagon/test_cumcount.py b/tests/triacontagon/test_cumcount.py index b9a8780..694b46c 100644 --- a/tests/triacontagon/test_cumcount.py +++ b/tests/triacontagon/test_cumcount.py @@ -1,26 +1,27 @@ from triacontagon.cumcount import dfill, \ argunsort, \ cumcount +import torch import numpy as np def test_dfill_01(): - input = np.array([1, 1, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5]) + input = torch.tensor([1, 1, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5]) output = dfill(input) - expected = np.array([0, 0, 0, 0, 0, 5, 5, 7, 7, 7, 10, 10, 12, 12]) - assert np.all(output == expected) + expected = torch.tensor([0, 0, 0, 0, 0, 5, 5, 7, 7, 7, 10, 10, 12, 12]) + assert torch.all(output == expected) def test_argunsort_01(): - input = np.array([1, 1, 2, 3, 3, 4, 1, 1, 5, 5, 2, 3, 1, 4]) - output = np.argsort(input, kind='mergesort') - output = argunsort(output) - expected = np.array([0, 1, 5, 7, 8, 10, 2, 3, 12, 13, 6, 9, 4, 11]) - assert np.all(output == expected) + input = torch.tensor([1, 1, 2, 3, 3, 4, 1, 1, 5, 5, 2, 3, 1, 4]) + output = np.argsort(input.numpy()) + output = argunsort(torch.tensor(output)) + expected = torch.tensor([0, 1, 5, 7, 8, 10, 2, 3, 12, 13, 6, 9, 4, 11]) + assert torch.all(output == expected) def test_cumcount_01(): - input = np.array([1, 1, 2, 3, 3, 4, 1, 1, 5, 5, 2, 3, 1, 4]) + input = torch.tensor([1, 1, 2, 3, 3, 4, 1, 1, 5, 5, 2, 3, 1, 4]) output = cumcount(input) - expected = np.array([0, 1, 0, 0, 1, 0, 2, 3, 0, 1, 1, 2, 4, 1]) - assert np.all(output == expected) + expected = torch.tensor([0, 1, 0, 0, 1, 0, 2, 3, 0, 1, 1, 2, 4, 1]) + assert torch.all(output == expected) diff --git a/tests/triacontagon/test_sampling.py b/tests/triacontagon/test_sampling.py index 088cee3..618f172 100644 --- a/tests/triacontagon/test_sampling.py +++ b/tests/triacontagon/test_sampling.py @@ -3,7 +3,8 @@ from triacontagon.sampling import fixed_unigram_candidate_sampler, \ get_true_classes, \ negative_sample_adj_mat, \ negative_sample_data, \ - get_edges_and_degrees + get_edges_and_degrees, \ + fixed_unigram_candidate_sampler_new from triacontagon.decode import dedicom_decoder import torch import time @@ -113,3 +114,12 @@ def test_negative_sample_data_01(): ], dedicom_decoder) d_neg = negative_sample_data(d) + + +def test_fixed_unigram_candidate_sampler_new_01(): + x = (torch.rand((10, 10)) < .1).to(torch.float32).to_sparse() + true_classes, row_count = get_true_classes(x) + edges, degrees = get_edges_and_degrees(x) + # import pdb + # pdb.set_trace() + _ = fixed_unigram_candidate_sampler_new(true_classes, row_count, degrees, 0.75) -- 2.26.2 From 670237c3f8bc6796d65cc83155ebfe474393b157 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 21 Aug 2020 17:15:58 +0200 Subject: [PATCH 211/227] Work on fixed_unigram_candidate_sampler_new(). --- src/triacontagon/cumcount.py | 1 + src/triacontagon/sampling.py | 6 +++--- tests/triacontagon/test_sampling.py | 10 +++++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/triacontagon/cumcount.py b/src/triacontagon/cumcount.py index faee2be..5c88da6 100644 --- a/src/triacontagon/cumcount.py +++ b/src/triacontagon/cumcount.py @@ -9,6 +9,7 @@ def dfill(a): torch.nonzero(a[:-1] != a[1:], as_tuple=True)[0] + 1, torch.tensor([n]) ]) + print('b:',b) res = torch.arange(n)[b[:-1]] res = torch.repeat_interleave(res, b[1:] - b[:-1]) return res diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py index 76e42dc..d43afd0 100644 --- a/src/triacontagon/sampling.py +++ b/src/triacontagon/sampling.py @@ -21,7 +21,7 @@ from itertools import product, \ from functools import reduce -def fixed_unigram_candidate_sampler_new( +def fixed_unigram_candidate_sampler( true_classes: torch.Tensor, num_repeats: torch.Tensor, unigrams: torch.Tensor, @@ -50,7 +50,7 @@ def fixed_unigram_candidate_sampler_new( dtype=true_classes.dtype) ], dim=1) - indices = torch.repeat_interleave(torch.arange(len(unigrams)), num_repeats) + indices = torch.repeat_interleave(torch.arange(len(true_classes)), num_repeats) indices = torch.cat([ torch.arange(len(indices)).view(-1, 1), indices.view(-1, 1) ], dim=1) @@ -135,7 +135,7 @@ def fixed_unigram_candidate_sampler_slow( return torch.tensor(res) -def fixed_unigram_candidate_sampler( +def fixed_unigram_candidate_sampler_old( true_classes: torch.Tensor, num_repeats: torch.Tensor, unigrams: torch.Tensor, diff --git a/tests/triacontagon/test_sampling.py b/tests/triacontagon/test_sampling.py index 618f172..d5480fb 100644 --- a/tests/triacontagon/test_sampling.py +++ b/tests/triacontagon/test_sampling.py @@ -3,11 +3,12 @@ from triacontagon.sampling import fixed_unigram_candidate_sampler, \ get_true_classes, \ negative_sample_adj_mat, \ negative_sample_data, \ - get_edges_and_degrees, \ - fixed_unigram_candidate_sampler_new + get_edges_and_degrees +import triacontagon.sampling from triacontagon.decode import dedicom_decoder import torch import time +import pytest def test_fixed_unigram_candidate_sampler_01(): @@ -41,6 +42,7 @@ def test_fixed_unigram_candidate_sampler_02(): print('row_count:', row_count) edges_pos, degrees = get_edges_and_degrees(bar_foo) + print('degrees:', degrees) res = fixed_unigram_candidate_sampler(true_classes, row_count, degrees, 0.75) @@ -117,7 +119,9 @@ def test_negative_sample_data_01(): def test_fixed_unigram_candidate_sampler_new_01(): - x = (torch.rand((10, 10)) < .1).to(torch.float32).to_sparse() + if 'fixed_unigram_candidate_sampler_new' not in dir(triacontagon.sampling): + pytest.skip('fixed_unigram_candidate_sampler_new not found') + x = (torch.rand((10, 10)) < .05).to(torch.float32).to_sparse() true_classes, row_count = get_true_classes(x) edges, degrees = get_edges_and_degrees(x) # import pdb -- 2.26.2 From 689dbeed20b219f18e9325b71464c1f386835807 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 21 Aug 2020 17:19:45 +0200 Subject: [PATCH 212/227] Add mergesort switch. --- src/triacontagon/cumcount.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/triacontagon/cumcount.py b/src/triacontagon/cumcount.py index 5c88da6..164c784 100644 --- a/src/triacontagon/cumcount.py +++ b/src/triacontagon/cumcount.py @@ -24,7 +24,7 @@ def argunsort(s): def cumcount(a): n = torch.numel(a) - s = np.argsort(a.detach().cpu().numpy()) + s = np.argsort(a.detach().cpu().numpy(), kind='mergesort') s = torch.tensor(s, device=a.device) i = argunsort(s) b = a[s] -- 2.26.2 From 490b4f9281e3e4af34167d665d076ab8f4396501 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 21 Aug 2020 17:24:07 +0200 Subject: [PATCH 213/227] New robust implementation of fixed_unigram_candidate_sampler() seems to be working. --- src/triacontagon/cumcount.py | 2 +- src/triacontagon/sampling.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/triacontagon/cumcount.py b/src/triacontagon/cumcount.py index 164c784..33169f9 100644 --- a/src/triacontagon/cumcount.py +++ b/src/triacontagon/cumcount.py @@ -9,7 +9,7 @@ def dfill(a): torch.nonzero(a[:-1] != a[1:], as_tuple=True)[0] + 1, torch.tensor([n]) ]) - print('b:',b) + # print('b:',b) res = torch.arange(n)[b[:-1]] res = torch.repeat_interleave(res, b[1:] - b[:-1]) return res diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py index d43afd0..3ae0ef3 100644 --- a/src/triacontagon/sampling.py +++ b/src/triacontagon/sampling.py @@ -77,14 +77,17 @@ def fixed_unigram_candidate_sampler( mask = mask | repeated updated = indices[~mask] - ofs = true_class_count[updated[:, 1]] + \ - cumcount(updated[:, 1]) - true_classes[updated[:, 1], ofs] = candidates[~mask].transpose(0, 1) - true_class_count[updated[:, 1]] = ofs + 1 + if len(updated) > 0: + ofs = true_class_count[updated[:, 1]] + \ + cumcount(updated[:, 1]) + true_classes[updated[:, 1], ofs] = candidates[~mask].transpose(0, 1) + true_class_count[updated[:, 1]] = ofs + 1 result[indices[:, 0]] = candidates.transpose(0, 1) indices = indices[mask] + return result + def fixed_unigram_candidate_sampler_slow( true_classes: torch.Tensor, -- 2.26.2 From d7d442c5e35ea54de7152f79e0b6e190e460877d Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 21 Aug 2020 17:33:13 +0200 Subject: [PATCH 214/227] New fixed_unigram_candidate_sampler() still requires work. --- src/triacontagon/sampling.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py index 3ae0ef3..b88f90d 100644 --- a/src/triacontagon/sampling.py +++ b/src/triacontagon/sampling.py @@ -21,7 +21,7 @@ from itertools import product, \ from functools import reduce -def fixed_unigram_candidate_sampler( +def fixed_unigram_candidate_sampler_new( true_classes: torch.Tensor, num_repeats: torch.Tensor, unigrams: torch.Tensor, @@ -57,6 +57,8 @@ def fixed_unigram_candidate_sampler( result = torch.zeros(len(indices), dtype=torch.long) while len(indices) > 0: + print(len(indices)) + candidates = torch.utils.data.WeightedRandomSampler(unigrams, len(indices)) candidates = torch.tensor(list(candidates)).view(-1, 1) @@ -73,6 +75,7 @@ def fixed_unigram_candidate_sampler( can_cum = cumcount(candidates[:, 0]) ind_cum = cumcount(indices[:, 1]) repeated = (can_cum > 0) & (ind_cum > 0) + # TODO: this is wrong, still requires work mask = mask | repeated @@ -138,7 +141,7 @@ def fixed_unigram_candidate_sampler_slow( return torch.tensor(res) -def fixed_unigram_candidate_sampler_old( +def fixed_unigram_candidate_sampler( true_classes: torch.Tensor, num_repeats: torch.Tensor, unigrams: torch.Tensor, -- 2.26.2 From 79b0890a704615c379d6f5d59622a69a4d5ed97d Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 21 Aug 2020 17:34:15 +0200 Subject: [PATCH 215/227] Mini fix. --- tests/triacontagon/test_sampling.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/triacontagon/test_sampling.py b/tests/triacontagon/test_sampling.py index d5480fb..981cf01 100644 --- a/tests/triacontagon/test_sampling.py +++ b/tests/triacontagon/test_sampling.py @@ -126,4 +126,5 @@ def test_fixed_unigram_candidate_sampler_new_01(): edges, degrees = get_edges_and_degrees(x) # import pdb # pdb.set_trace() - _ = fixed_unigram_candidate_sampler_new(true_classes, row_count, degrees, 0.75) + _ = triacontagon.sampling.fixed_unigram_candidate_sampler_new(true_classes, + row_count, degrees, 0.75) -- 2.26.2 From 356f3af3c38ef87ef40c59ae96f2e52def5e7f2d Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 21 Aug 2020 18:51:27 +0200 Subject: [PATCH 216/227] With this weird little trick the new fixed_unigram_candidate_sampler() finally seems to be fully robust and to actually work. --- src/triacontagon/sampling.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/triacontagon/sampling.py b/src/triacontagon/sampling.py index b88f90d..73d7cf2 100644 --- a/src/triacontagon/sampling.py +++ b/src/triacontagon/sampling.py @@ -21,7 +21,7 @@ from itertools import product, \ from functools import reduce -def fixed_unigram_candidate_sampler_new( +def fixed_unigram_candidate_sampler( true_classes: torch.Tensor, num_repeats: torch.Tensor, unigrams: torch.Tensor, @@ -72,9 +72,10 @@ def fixed_unigram_candidate_sampler_new( mask = (true_classes[indices[:, 1]] == candidates).sum(dim=1).to(torch.bool) - can_cum = cumcount(candidates[:, 0]) + # can_cum = cumcount(candidates[:, 0]) + can_diff = torch.cat([ torch.tensor([1]), candidates[1:, 0] - candidates[:-1, 0] ]) ind_cum = cumcount(indices[:, 1]) - repeated = (can_cum > 0) & (ind_cum > 0) + repeated = (can_diff == 0) & (ind_cum > 0) # TODO: this is wrong, still requires work mask = mask | repeated @@ -141,7 +142,7 @@ def fixed_unigram_candidate_sampler_slow( return torch.tensor(res) -def fixed_unigram_candidate_sampler( +def fixed_unigram_candidate_sampler_old( true_classes: torch.Tensor, num_repeats: torch.Tensor, unigrams: torch.Tensor, -- 2.26.2 From 37c5db1330bfa5666db7a30e664e1d6af1c7bc5d Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 21 Aug 2020 22:56:28 +0200 Subject: [PATCH 217/227] Start torch_stablesort.0 --- src/torch_stablesort/dispatch_test.cpp | 33 +++++ src/torch_stablesort/setup.py | 7 ++ src/torch_stablesort/torch_stablesort.cpp | 140 ++++++++++++++++++++++ 3 files changed, 180 insertions(+) create mode 100644 src/torch_stablesort/dispatch_test.cpp create mode 100644 src/torch_stablesort/setup.py create mode 100644 src/torch_stablesort/torch_stablesort.cpp diff --git a/src/torch_stablesort/dispatch_test.cpp b/src/torch_stablesort/dispatch_test.cpp new file mode 100644 index 0000000..8079998 --- /dev/null +++ b/src/torch_stablesort/dispatch_test.cpp @@ -0,0 +1,33 @@ +#include +#include + +template +class Dispatcher { +public: + void dispatch(int x, Ts&& ...args) { + if (x > 0) + call(std::forward(args)...); + else + call(std::forward(args)...); + } + +protected: + template + void call(Ts&& ...args) { + throw std::runtime_error("Not implemented"); + } +}; + +class bla {}; + +template<> template +void Dispatcher::call(int&& a, char&& b, double&& c) { + std::cout << typeid(T).name() << " " << a << " " << b << " " << c << std::endl; +} + +main() { + std::cout << "main()" << std::endl; + Dispatcher d; + d.dispatch(5, 1, 'a', 5.5); + d.dispatch(-5, 1, 'a', 5.5); +} diff --git a/src/torch_stablesort/setup.py b/src/torch_stablesort/setup.py new file mode 100644 index 0000000..3071c80 --- /dev/null +++ b/src/torch_stablesort/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup, Extension +from torch.utils import cpp_extension + +setup(name='torch_stablesort', + ext_modules=[cpp_extension.CppExtension('torch_stablesort_cpp', + ['torch_stablesort.cpp'])], + cmdclass={'build_ext': cpp_extension.BuildExtension}) diff --git a/src/torch_stablesort/torch_stablesort.cpp b/src/torch_stablesort/torch_stablesort.cpp new file mode 100644 index 0000000..d9cd856 --- /dev/null +++ b/src/torch_stablesort/torch_stablesort.cpp @@ -0,0 +1,140 @@ +#include + +#include +#include +#include + + +template +void dispatch(torch::Tensor input, Ts&& ... args) { + switch(input.type().scalarType()) { + case torch::ScalarType::Double: + return fun(input, std::forward(args)...); + case torch::ScalarType::Float: + return fun(input, std::forward(args)...); + case torch::ScalarType::Half: + throw std::runtime_error("Half-precision float not supported"); + case torch::ScalarType::ComplexHalf: + throw std::runtime_error("Half-precision complex float not supported"); + case torch::ScalarType::ComplexFloat: + return fun(input, std::forward(args)...); + case torch::ScalarType::ComplexDouble: + return fun(input, std::forward(args)...); + case torch::ScalarType::Long: + return fun(input, std::forward(args)...); + case torch::ScalarType::Int: + return fun(input, std::forward(args)...); + case torch::ScalarType::Short: + return fun(input, std::forward(args)...); + case torch::ScalarType::Char: + return fun(input, std::forward(args)...); + case torch::ScalarType::Byte: + return fun(input, std::forward(args)...); + case torch::ScalarType::Bool: + return fun(input, std::forward(args)...); + case torch::ScalarType::QInt32: + throw std::runtime_error("QInt32 not supported"); + case torch::ScalarType::QInt16: + throw std::runtime_error("QInt16 not supported"); + case torch::ScalarType::QInt8: + throw std::runtime_error("QInt8 not supported"); + case torch::ScalarType::BFloat16: + throw std::runtime_error("BFloat16 not supported"); + default: + throw std::runtime_error("Unknown scalar type"); + } +} + + +std::vector stable_sort_forward( + torch::Tensor input, + int dim, + bool descending, + torch::optional out = nullptr) { + + + + auto X = torch::cat({old_h, input}, /*dim=*/1); + + auto gate_weights = torch::addmm(bias, X, weights.transpose(0, 1)); + auto gates = gate_weights.chunk(3, /*dim=*/1); + + auto input_gate = torch::sigmoid(gates[0]); + auto output_gate = torch::sigmoid(gates[1]); + auto candidate_cell = torch::elu(gates[2], /*alpha=*/1.0); + + auto new_cell = old_cell + candidate_cell * input_gate; + auto new_h = torch::tanh(new_cell) * output_gate; + + return {new_h, + new_cell, + input_gate, + output_gate, + candidate_cell, + X, + gate_weights}; +} + +/ tanh'(z) = 1 - tanh^2(z) +torch::Tensor d_tanh(torch::Tensor z) { + return 1 - z.tanh().pow(2); +} + +// elu'(z) = relu'(z) + { alpha * exp(z) if (alpha * (exp(z) - 1)) < 0, else 0} +torch::Tensor d_elu(torch::Tensor z, torch::Scalar alpha = 1.0) { + auto e = z.exp(); + auto mask = (alpha * (e - 1)) < 0; + return (z > 0).type_as(z) + mask.type_as(z) * (alpha * e); +} + +std::vector stable_sort_backward( + torch::Tensor grad_h, + torch::Tensor grad_cell, + torch::Tensor new_cell, + torch::Tensor input_gate, + torch::Tensor output_gate, + torch::Tensor candidate_cell, + torch::Tensor X, + torch::Tensor gate_weights, + torch::Tensor weights) { + auto d_output_gate = torch::tanh(new_cell) * grad_h; + auto d_tanh_new_cell = output_gate * grad_h; + auto d_new_cell = d_tanh(new_cell) * d_tanh_new_cell + grad_cell; + + auto d_old_cell = d_new_cell; + auto d_candidate_cell = input_gate * d_new_cell; + auto d_input_gate = candidate_cell * d_new_cell; + + auto gates = gate_weights.chunk(3, /*dim=*/1); + d_input_gate *= d_sigmoid(gates[0]); + d_output_gate *= d_sigmoid(gates[1]); + d_candidate_cell *= d_elu(gates[2]); + + auto d_gates = + torch::cat({d_input_gate, d_output_gate, d_candidate_cell}, /*dim=*/1); + + auto d_weights = d_gates.t().mm(X); + auto d_bias = d_gates.sum(/*dim=*/0, /*keepdim=*/true); + + auto d_X = d_gates.mm(weights); + const auto state_size = grad_h.size(1); + auto d_old_h = d_X.slice(/*dim=*/1, 0, state_size); + auto d_input = d_X.slice(/*dim=*/1, state_size); + + return {d_old_h, d_input, d_weights, d_bias, d_old_cell}; +} + +std::vector stable_argsort_forward() { + +} + +std::vector stable_argsort_backward() { + +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("stable_sort_forward", &stable_sort_forward, "Stable sort forward"); + m.def("stable_sort_backward", &stable_sort_backward, "Stable sort backward"); + m.def("stable_argsort_forward", &stable_argsort_forward, "Stable argsort forward"); + m.def("stable_argsort_backward", &stable_argsort_backward, "Stable argsort backward"); +} -- 2.26.2 From 14321daf624b1359825674da0b4b49a51b94784c Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 21 Aug 2020 23:31:02 +0200 Subject: [PATCH 218/227] Improved dispatch. --- src/torch_stablesort/dispatch_test.cpp | 36 ++++++++++---------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/src/torch_stablesort/dispatch_test.cpp b/src/torch_stablesort/dispatch_test.cpp index 8079998..5756088 100644 --- a/src/torch_stablesort/dispatch_test.cpp +++ b/src/torch_stablesort/dispatch_test.cpp @@ -1,33 +1,25 @@ #include #include -template -class Dispatcher { -public: - void dispatch(int x, Ts&& ...args) { - if (x > 0) - call(std::forward(args)...); - else - call(std::forward(args)...); - } +template class F, typename... Ts> +void dispatch(int x, Ts&& ...args) { + if (x > 0) + F()(std::forward(args)...); + else + F()(std::forward(args)...); +} -protected: - template - void call(Ts&& ...args) { - throw std::runtime_error("Not implemented"); +template +struct bla { + void operator()(int&& a, char&& b, double&& c) const { + std::cout << typeid(T).name() << " " << a << " " << b << " " << c << std::endl; } }; -class bla {}; - -template<> template -void Dispatcher::call(int&& a, char&& b, double&& c) { - std::cout << typeid(T).name() << " " << a << " " << b << " " << c << std::endl; -} main() { std::cout << "main()" << std::endl; - Dispatcher d; - d.dispatch(5, 1, 'a', 5.5); - d.dispatch(-5, 1, 'a', 5.5); + //bla()(1, 'a', 5.5); + dispatch(5, 1, 'a', 5.5); + dispatch(-5, 1, 'a', 5.5); } -- 2.26.2 From 747156a97152e5e78ceec8090afa441288bdfbf7 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 24 Aug 2020 10:21:06 +0200 Subject: [PATCH 219/227] torch_stablesort_cpp seems to work. --- .gitignore | 4 + src/torch_stablesort/dispatch.h | 41 ++++ src/torch_stablesort/dispatch_test.cpp | 10 +- src/torch_stablesort/openmp_test.cpp | 8 + src/torch_stablesort/setup.py | 3 +- src/torch_stablesort/torch_stablesort.cpp | 225 +++++++++++----------- 6 files changed, 175 insertions(+), 116 deletions(-) create mode 100644 src/torch_stablesort/dispatch.h create mode 100644 src/torch_stablesort/openmp_test.cpp diff --git a/.gitignore b/.gitignore index 9ec7cab..63adf45 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ __pycache__ /docs/icosagon/*.png /experiments/decagon_run/profiler_results /experiments/decagon_run_effcat/profiler_results +/src/torch_stablesort/dist +/src/torch_stablesort/build +/src/torch_stablesort/torch_stablesort.egg-info +a.out diff --git a/src/torch_stablesort/dispatch.h b/src/torch_stablesort/dispatch.h new file mode 100644 index 0000000..1dc9b49 --- /dev/null +++ b/src/torch_stablesort/dispatch.h @@ -0,0 +1,41 @@ +#include + +template class F, typename R, typename... Ts> +R dispatch(torch::Tensor input, Ts&& ... args) { + switch(input.type().scalarType()) { + case torch::ScalarType::Double: + return F()(input, std::forward(args)...); + case torch::ScalarType::Float: + return F()(input, std::forward(args)...); + case torch::ScalarType::Half: + throw std::runtime_error("Half-precision float not supported"); + case torch::ScalarType::ComplexHalf: + throw std::runtime_error("Half-precision complex float not supported"); + case torch::ScalarType::ComplexFloat: + throw std::runtime_error("Complex float not supported"); + case torch::ScalarType::ComplexDouble: + throw std::runtime_error("Complex double not supported"); + case torch::ScalarType::Long: + return F()(input, std::forward(args)...); + case torch::ScalarType::Int: + return F()(input, std::forward(args)...); + case torch::ScalarType::Short: + return F()(input, std::forward(args)...); + case torch::ScalarType::Char: + return F()(input, std::forward(args)...); + case torch::ScalarType::Byte: + return F()(input, std::forward(args)...); + case torch::ScalarType::Bool: + return F()(input, std::forward(args)...); + case torch::ScalarType::QInt32: + throw std::runtime_error("QInt32 not supported"); + //case torch::ScalarType::QInt16: + // throw std::runtime_error("QInt16 not supported"); + case torch::ScalarType::QInt8: + throw std::runtime_error("QInt8 not supported"); + case torch::ScalarType::BFloat16: + throw std::runtime_error("BFloat16 not supported"); + default: + throw std::runtime_error("Unknown scalar type"); + } +} diff --git a/src/torch_stablesort/dispatch_test.cpp b/src/torch_stablesort/dispatch_test.cpp index 5756088..76e043f 100644 --- a/src/torch_stablesort/dispatch_test.cpp +++ b/src/torch_stablesort/dispatch_test.cpp @@ -12,14 +12,20 @@ void dispatch(int x, Ts&& ...args) { template struct bla { void operator()(int&& a, char&& b, double&& c) const { - std::cout << typeid(T).name() << " " << a << " " << b << " " << c << std::endl; + std::cout << sizeof(T) << " " << typeid(T).name() << " " << a << " " << b << " " << c << std::endl; } }; +template +struct bla128 { + void operator()(int&& a, char&& b, __float128&& c) const { + std::cout << sizeof(T) << " " << typeid(T).name() << " " << a << " " << b << " " << (double) c << std::endl; + } +}; main() { std::cout << "main()" << std::endl; //bla()(1, 'a', 5.5); - dispatch(5, 1, 'a', 5.5); + dispatch(5, 1, 'a', (__float128) 5.5); dispatch(-5, 1, 'a', 5.5); } diff --git a/src/torch_stablesort/openmp_test.cpp b/src/torch_stablesort/openmp_test.cpp new file mode 100644 index 0000000..10b364e --- /dev/null +++ b/src/torch_stablesort/openmp_test.cpp @@ -0,0 +1,8 @@ +#include + +main() { + #pragma omp parallel for + for (int i = 0; i < 10; i++) { + std::cout << i << std::endl; + } +} diff --git a/src/torch_stablesort/setup.py b/src/torch_stablesort/setup.py index 3071c80..163d8ff 100644 --- a/src/torch_stablesort/setup.py +++ b/src/torch_stablesort/setup.py @@ -3,5 +3,6 @@ from torch.utils import cpp_extension setup(name='torch_stablesort', ext_modules=[cpp_extension.CppExtension('torch_stablesort_cpp', - ['torch_stablesort.cpp'])], + ['torch_stablesort.cpp'], + extra_compile_args=['-fopenmp', '-ggdb'])], cmdclass={'build_ext': cpp_extension.BuildExtension}) diff --git a/src/torch_stablesort/torch_stablesort.cpp b/src/torch_stablesort/torch_stablesort.cpp index d9cd856..94b3e13 100644 --- a/src/torch_stablesort/torch_stablesort.cpp +++ b/src/torch_stablesort/torch_stablesort.cpp @@ -4,137 +4,136 @@ #include #include +#include "dispatch.h" -template -void dispatch(torch::Tensor input, Ts&& ... args) { - switch(input.type().scalarType()) { - case torch::ScalarType::Double: - return fun(input, std::forward(args)...); - case torch::ScalarType::Float: - return fun(input, std::forward(args)...); - case torch::ScalarType::Half: - throw std::runtime_error("Half-precision float not supported"); - case torch::ScalarType::ComplexHalf: - throw std::runtime_error("Half-precision complex float not supported"); - case torch::ScalarType::ComplexFloat: - return fun(input, std::forward(args)...); - case torch::ScalarType::ComplexDouble: - return fun(input, std::forward(args)...); - case torch::ScalarType::Long: - return fun(input, std::forward(args)...); - case torch::ScalarType::Int: - return fun(input, std::forward(args)...); - case torch::ScalarType::Short: - return fun(input, std::forward(args)...); - case torch::ScalarType::Char: - return fun(input, std::forward(args)...); - case torch::ScalarType::Byte: - return fun(input, std::forward(args)...); - case torch::ScalarType::Bool: - return fun(input, std::forward(args)...); - case torch::ScalarType::QInt32: - throw std::runtime_error("QInt32 not supported"); - case torch::ScalarType::QInt16: - throw std::runtime_error("QInt16 not supported"); - case torch::ScalarType::QInt8: - throw std::runtime_error("QInt8 not supported"); - case torch::ScalarType::BFloat16: - throw std::runtime_error("BFloat16 not supported"); - default: - throw std::runtime_error("Unknown scalar type"); - } -} +template +struct stable_sort_impl { + std::vector operator()( + torch::Tensor input, + int dim, + bool descending, + torch::optional> out + ) const { + if (input.is_sparse()) + throw std::runtime_error("Sparse tensors are not supported"); -std::vector stable_sort_forward( - torch::Tensor input, - int dim, - bool descending, - torch::optional out = nullptr) { + if (input.device().type() != torch::DeviceType::CPU) + throw std::runtime_error("Only CPU tensors are supported"); + if (out != torch::nullopt) + throw std::runtime_error("out argument is not supported"); + auto in = (dim != -1) ? + torch::transpose(input, dim, -1) : + input; - auto X = torch::cat({old_h, input}, /*dim=*/1); + auto in_sizes = in.sizes(); - auto gate_weights = torch::addmm(bias, X, weights.transpose(0, 1)); - auto gates = gate_weights.chunk(3, /*dim=*/1); + // std::cout << "in_sizes: " << in_sizes << std::endl; - auto input_gate = torch::sigmoid(gates[0]); - auto output_gate = torch::sigmoid(gates[1]); - auto candidate_cell = torch::elu(gates[2], /*alpha=*/1.0); + in = in.view({ -1, in.size(-1) }).contiguous(); - auto new_cell = old_cell + candidate_cell * input_gate; - auto new_h = torch::tanh(new_cell) * output_gate; + auto in_outer_stride = in.stride(-2); + auto in_inner_stride = in.stride(-1); - return {new_h, - new_cell, - input_gate, - output_gate, - candidate_cell, - X, - gate_weights}; -} + auto pin = static_cast(in.data_ptr()); -/ tanh'(z) = 1 - tanh^2(z) -torch::Tensor d_tanh(torch::Tensor z) { - return 1 - z.tanh().pow(2); -} + auto x = in.clone(); -// elu'(z) = relu'(z) + { alpha * exp(z) if (alpha * (exp(z) - 1)) < 0, else 0} -torch::Tensor d_elu(torch::Tensor z, torch::Scalar alpha = 1.0) { - auto e = z.exp(); - auto mask = (alpha * (e - 1)) < 0; - return (z > 0).type_as(z) + mask.type_as(z) * (alpha * e); -} + auto x_outer_stride = x.stride(-2); + auto x_inner_stride = x.stride(-1); -std::vector stable_sort_backward( - torch::Tensor grad_h, - torch::Tensor grad_cell, - torch::Tensor new_cell, - torch::Tensor input_gate, - torch::Tensor output_gate, - torch::Tensor candidate_cell, - torch::Tensor X, - torch::Tensor gate_weights, - torch::Tensor weights) { - auto d_output_gate = torch::tanh(new_cell) * grad_h; - auto d_tanh_new_cell = output_gate * grad_h; - auto d_new_cell = d_tanh(new_cell) * d_tanh_new_cell + grad_cell; - - auto d_old_cell = d_new_cell; - auto d_candidate_cell = input_gate * d_new_cell; - auto d_input_gate = candidate_cell * d_new_cell; - - auto gates = gate_weights.chunk(3, /*dim=*/1); - d_input_gate *= d_sigmoid(gates[0]); - d_output_gate *= d_sigmoid(gates[1]); - d_candidate_cell *= d_elu(gates[2]); - - auto d_gates = - torch::cat({d_input_gate, d_output_gate, d_candidate_cell}, /*dim=*/1); - - auto d_weights = d_gates.t().mm(X); - auto d_bias = d_gates.sum(/*dim=*/0, /*keepdim=*/true); - - auto d_X = d_gates.mm(weights); - const auto state_size = grad_h.size(1); - auto d_old_h = d_X.slice(/*dim=*/1, 0, state_size); - auto d_input = d_X.slice(/*dim=*/1, state_size); - - return {d_old_h, d_input, d_weights, d_bias, d_old_cell}; -} + auto n_cols = x.size(1); + auto n_rows = x.size(0); + auto px = static_cast(x.data_ptr()); -std::vector stable_argsort_forward() { + auto y = torch::empty({ n_rows, n_cols }, + torch::TensorOptions().dtype(torch::kInt64)); -} + auto y_outer_stride = y.stride(-2); + auto y_inner_stride = y.stride(-1); + + auto py = static_cast(y.data_ptr()); + + if (descending) { + #pragma omp parallel for + for (decltype(n_rows) i = 0; i < n_rows; i++) { + + std::vector indices(n_cols); + for (decltype(n_cols) k = 0; k < n_cols; k++) { + indices[k] = k; + } + + std::stable_sort(std::begin(indices), std::end(indices), + [pin, i, in_outer_stride, in_inner_stride](const auto &a, const auto &b) { + auto va = pin[i * in_outer_stride + a * in_inner_stride]; + auto vb = pin[i * in_outer_stride + b * in_inner_stride]; + return (vb < va); + }); + + for (decltype(n_cols) k = 0; k < n_cols; k++) { + py[i * y_outer_stride + k * y_inner_stride] = indices[k]; + px[i * x_outer_stride + k * x_inner_stride] = + pin[i * in_outer_stride + indices[k] * in_inner_stride]; + } + } -std::vector stable_argsort_backward() { + } else { + #pragma omp parallel for + for (decltype(n_rows) i = 0; i < n_rows; i++) { + + std::vector indices(n_cols); + for (decltype(n_cols) k = 0; k < n_cols; k++) { + indices[k] = k; + } + + std::stable_sort(std::begin(indices), std::end(indices), + [pin, i, in_outer_stride, in_inner_stride](const auto &a, const auto &b) { + auto va = pin[i * in_outer_stride + a * in_inner_stride]; + auto vb = pin[i * in_outer_stride + b * in_inner_stride]; + return (va < vb); + }); + + for (decltype(n_cols) k = 0; k < n_cols; k++) { + py[i * y_outer_stride + k * y_inner_stride] = indices[k]; + px[i * x_outer_stride + k * x_inner_stride] = + pin[i * in_outer_stride + indices[k] * in_inner_stride]; + } + } + } + + // std::cout << "Here" << std::endl; + + x = x.view(in_sizes); + y = y.view(in_sizes); + + x = (dim == -1) ? + x : + torch::transpose(x, dim, -1).contiguous(); + + y = (dim == -1) ? + y : + torch::transpose(y, dim, -1).contiguous(); + + // std::cout << "Here 2" << std::endl; + + return { x, y }; + } +}; + +std::vector stable_sort( + torch::Tensor input, + int dim = -1, + bool descending = false, + torch::optional> out = torch::nullopt) { + return dispatch>( + input, dim, descending, out); } PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { - m.def("stable_sort_forward", &stable_sort_forward, "Stable sort forward"); - m.def("stable_sort_backward", &stable_sort_backward, "Stable sort backward"); - m.def("stable_argsort_forward", &stable_argsort_forward, "Stable argsort forward"); - m.def("stable_argsort_backward", &stable_argsort_backward, "Stable argsort backward"); + m.def("stable_sort", &stable_sort, "Stable sort", + py::arg("input"), py::arg("dim") = -1, py::arg("descending") = false, + py::arg("out") = nullptr); } -- 2.26.2 From 87953842a63d7b12319ab108e5f31e2ee6c091aa Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 24 Aug 2020 13:28:53 +0200 Subject: [PATCH 220/227] Try to implement gradient. --- src/torch_stablesort/setup.py | 1 + src/torch_stablesort/torch_stablesort.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 src/torch_stablesort/torch_stablesort.py diff --git a/src/torch_stablesort/setup.py b/src/torch_stablesort/setup.py index 163d8ff..a45a47f 100644 --- a/src/torch_stablesort/setup.py +++ b/src/torch_stablesort/setup.py @@ -2,6 +2,7 @@ from setuptools import setup, Extension from torch.utils import cpp_extension setup(name='torch_stablesort', + py_modules=['torch_stablesort'], ext_modules=[cpp_extension.CppExtension('torch_stablesort_cpp', ['torch_stablesort.cpp'], extra_compile_args=['-fopenmp', '-ggdb'])], diff --git a/src/torch_stablesort/torch_stablesort.py b/src/torch_stablesort/torch_stablesort.py new file mode 100644 index 0000000..4a171d0 --- /dev/null +++ b/src/torch_stablesort/torch_stablesort.py @@ -0,0 +1,18 @@ +import torch +import torch_stablesort_cpp + + +class StableSort(torch.autograd.Function): + @staticmethod + def forward(ctx, input, dim=-1, descending=False, out=None): + values, indices = \ + torch_stablesort_cpp.stable_sort(input, dim, descending, out) + ctx.save_for_backward(input, indices) + return values, indices + + @staticmethod + def backward(ctx, grad_values, grad_indices): + input, indices = ctx.saved_variables + res = torch.empty_like(grad_values) + res[indices] = grad_values + grad_indices + return res -- 2.26.2 From 283d9387a8549a16bec6a60a7e0934f97d9f1ae4 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 24 Aug 2020 18:31:26 +0200 Subject: [PATCH 221/227] Add torch_stablesort.md. --- src/torch_stablesort/torch_stablesort.md | 29 ++++++++++++++++++++++++ src/torch_stablesort/torch_stablesort.py | 20 ++++++++++++---- 2 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 src/torch_stablesort/torch_stablesort.md diff --git a/src/torch_stablesort/torch_stablesort.md b/src/torch_stablesort/torch_stablesort.md new file mode 100644 index 0000000..e15d2ee --- /dev/null +++ b/src/torch_stablesort/torch_stablesort.md @@ -0,0 +1,29 @@ +# torch_stablesort + +## Introduction + +### Stable sorting algorithms + +Stable sort algorithms sort repeated elements in the same order that they appear in the input. When sorting some kinds of data, only part of the data is examined when determining the sort order. For example, in the card sorting example to the right, the cards are being sorted by their rank, and their suit is being ignored. This allows the possibility of multiple different correctly sorted versions of the original list. Stable sorting algorithms choose one of these, according to the following rule: if two items compare as equal, like the two 5 cards, then their relative order will be preserved, so that if one came before the other in the input, it will also come before the other in the output. + +### PyTorch + +PyTorch is an open source machine learning library based on the Torch library, used for applications such as computer vision and natural language processing, primarily developed by Facebook's AI Research lab. It is free and open-source software released under the Modified BSD license. + +### PyTorch Extensions + +PyTorch provides a plethora of operations related to neural networks, arbitrary tensor algebra, data wrangling and other purposes. However, you may still find yourself in need of a more customized operation. For example, you might want to use a novel activation function you found in a paper, or implement an operation you developed as part of your research. + +The easiest way of integrating such a custom operation in PyTorch is to write it in Python by extending Function and Module as outlined here. This gives you the full power of automatic differentiation (spares you from writing derivative functions) as well as the usual expressiveness of Python. However, there may be times when your operation is better implemented in C++. For example, your code may need to be really fast because it is called very frequently in your model or is very expensive even for few calls. Another plausible reason is that it depends on or interacts with other C or C++ libraries. To address such cases, PyTorch provides a very easy way of writing custom C++ extensions. + +## Implementation + +### setup.py + + + +### dispatch.h + +### torch_stablesort.cpp + +### torch_stablesort.py diff --git a/src/torch_stablesort/torch_stablesort.py b/src/torch_stablesort/torch_stablesort.py index 4a171d0..83b7574 100644 --- a/src/torch_stablesort/torch_stablesort.py +++ b/src/torch_stablesort/torch_stablesort.py @@ -7,12 +7,22 @@ class StableSort(torch.autograd.Function): def forward(ctx, input, dim=-1, descending=False, out=None): values, indices = \ torch_stablesort_cpp.stable_sort(input, dim, descending, out) - ctx.save_for_backward(input, indices) - return values, indices + ctx.save_for_backward(input, indices, torch.tensor(dim)) + return values, indices.detach() @staticmethod def backward(ctx, grad_values, grad_indices): - input, indices = ctx.saved_variables - res = torch.empty_like(grad_values) - res[indices] = grad_values + grad_indices + input, indices, dim = ctx.saved_variables + # print('backward(), grad_indices:', grad_indices, 'indices:', indices, + # 'grad_values:', grad_values) + + res = torch.gather(grad_values, dim, indices) + + # res = torch.empty_like(grad_values) + # print('here') + # res = res.view(-1, res.size(-1)) + # indices = indices.view(-1, indices.size(-1)) + # torch.repeat_interleave(torch.arange(indices.size(0)) + # res[indices] = grad_values # + grad_indices + # print('here 2') return res -- 2.26.2 From b439a46fc33fd4b206aa7f1a71aeaba8404248ea Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Fri, 28 Aug 2020 22:31:44 +0200 Subject: [PATCH 222/227] Use if constexpr to make stable_sort_impl() more compact. --- src/torch_stablesort/setup.py | 2 +- src/torch_stablesort/torch_stablesort.cpp | 74 +++++++++-------------- 2 files changed, 31 insertions(+), 45 deletions(-) diff --git a/src/torch_stablesort/setup.py b/src/torch_stablesort/setup.py index a45a47f..cf5a856 100644 --- a/src/torch_stablesort/setup.py +++ b/src/torch_stablesort/setup.py @@ -5,5 +5,5 @@ setup(name='torch_stablesort', py_modules=['torch_stablesort'], ext_modules=[cpp_extension.CppExtension('torch_stablesort_cpp', ['torch_stablesort.cpp'], - extra_compile_args=['-fopenmp', '-ggdb'])], + extra_compile_args=['-fopenmp', '-ggdb', '-std=c++1z'])], cmdclass={'build_ext': cpp_extension.BuildExtension}) diff --git a/src/torch_stablesort/torch_stablesort.cpp b/src/torch_stablesort/torch_stablesort.cpp index 94b3e13..4bc3d6e 100644 --- a/src/torch_stablesort/torch_stablesort.cpp +++ b/src/torch_stablesort/torch_stablesort.cpp @@ -6,12 +6,11 @@ #include "dispatch.h" -template +template struct stable_sort_impl { std::vector operator()( torch::Tensor input, int dim, - bool descending, torch::optional> out ) const { @@ -56,50 +55,27 @@ struct stable_sort_impl { auto py = static_cast(y.data_ptr()); - if (descending) { - #pragma omp parallel for - for (decltype(n_rows) i = 0; i < n_rows; i++) { - - std::vector indices(n_cols); - for (decltype(n_cols) k = 0; k < n_cols; k++) { - indices[k] = k; - } - - std::stable_sort(std::begin(indices), std::end(indices), - [pin, i, in_outer_stride, in_inner_stride](const auto &a, const auto &b) { - auto va = pin[i * in_outer_stride + a * in_inner_stride]; - auto vb = pin[i * in_outer_stride + b * in_inner_stride]; - return (vb < va); - }); - - for (decltype(n_cols) k = 0; k < n_cols; k++) { - py[i * y_outer_stride + k * y_inner_stride] = indices[k]; - px[i * x_outer_stride + k * x_inner_stride] = - pin[i * in_outer_stride + indices[k] * in_inner_stride]; - } + #pragma omp parallel for + for (decltype(n_rows) i = 0; i < n_rows; i++) { + std::vector indices(n_cols); + for (decltype(n_cols) k = 0; k < n_cols; k++) { + indices[k] = k; } - } else { - #pragma omp parallel for - for (decltype(n_rows) i = 0; i < n_rows; i++) { - - std::vector indices(n_cols); - for (decltype(n_cols) k = 0; k < n_cols; k++) { - indices[k] = k; - } - - std::stable_sort(std::begin(indices), std::end(indices), - [pin, i, in_outer_stride, in_inner_stride](const auto &a, const auto &b) { - auto va = pin[i * in_outer_stride + a * in_inner_stride]; - auto vb = pin[i * in_outer_stride + b * in_inner_stride]; + std::stable_sort(std::begin(indices), std::end(indices), + [pin, i, in_outer_stride, in_inner_stride](const auto &a, const auto &b) { + auto va = pin[i * in_outer_stride + a * in_inner_stride]; + auto vb = pin[i * in_outer_stride + b * in_inner_stride]; + if constexpr(descending) + return (vb < va); + else return (va < vb); - }); + }); - for (decltype(n_cols) k = 0; k < n_cols; k++) { - py[i * y_outer_stride + k * y_inner_stride] = indices[k]; - px[i * x_outer_stride + k * x_inner_stride] = - pin[i * in_outer_stride + indices[k] * in_inner_stride]; - } + for (decltype(n_cols) k = 0; k < n_cols; k++) { + py[i * y_outer_stride + k * y_inner_stride] = indices[k]; + px[i * x_outer_stride + k * x_inner_stride] = + pin[i * in_outer_stride + indices[k] * in_inner_stride]; } } @@ -122,14 +98,24 @@ struct stable_sort_impl { } }; +template +struct stable_sort_impl_desc: stable_sort_impl {}; + +template +struct stable_sort_impl_asc: stable_sort_impl {}; + std::vector stable_sort( torch::Tensor input, int dim = -1, bool descending = false, torch::optional> out = torch::nullopt) { - return dispatch>( - input, dim, descending, out); + if (descending) + return dispatch>( + input, dim, out); + else + return dispatch>( + input, dim, out); } PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { -- 2.26.2 From f8f2901eec5d3d421aaf5569431335fb2e8bacd2 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Mon, 31 Aug 2020 20:40:55 +0200 Subject: [PATCH 223/227] Start working on CUDA implementation of torch_stablesort. --- src/torch_stablesort/dispatch.h | 2 + src/torch_stablesort/setup.py | 5 +- src/torch_stablesort/torch_stablesort.cpp | 115 ++---------------- src/torch_stablesort/torch_stablesort_cpu.h | 119 +++++++++++++++++++ src/torch_stablesort/torch_stablesort_cuda.h | 109 +++++++++++++++++ 5 files changed, 243 insertions(+), 107 deletions(-) create mode 100644 src/torch_stablesort/torch_stablesort_cpu.h create mode 100644 src/torch_stablesort/torch_stablesort_cuda.h diff --git a/src/torch_stablesort/dispatch.h b/src/torch_stablesort/dispatch.h index 1dc9b49..a7dfb59 100644 --- a/src/torch_stablesort/dispatch.h +++ b/src/torch_stablesort/dispatch.h @@ -1,3 +1,5 @@ +#pragma once + #include template class F, typename R, typename... Ts> diff --git a/src/torch_stablesort/setup.py b/src/torch_stablesort/setup.py index cf5a856..8001773 100644 --- a/src/torch_stablesort/setup.py +++ b/src/torch_stablesort/setup.py @@ -5,5 +5,6 @@ setup(name='torch_stablesort', py_modules=['torch_stablesort'], ext_modules=[cpp_extension.CppExtension('torch_stablesort_cpp', ['torch_stablesort.cpp'], - extra_compile_args=['-fopenmp', '-ggdb', '-std=c++1z'])], - cmdclass={'build_ext': cpp_extension.BuildExtension}) + extra_compile_args=['-I/pstore/home/adaszews/scratch/thrust', + '-fopenmp', '-ggdb', '-std=c++1z'])], + cmdclass={'build_ext': cpp_extension.BuildExtension}) diff --git a/src/torch_stablesort/torch_stablesort.cpp b/src/torch_stablesort/torch_stablesort.cpp index 4bc3d6e..bbcac3c 100644 --- a/src/torch_stablesort/torch_stablesort.cpp +++ b/src/torch_stablesort/torch_stablesort.cpp @@ -4,105 +4,8 @@ #include #include -#include "dispatch.h" - -template -struct stable_sort_impl { - std::vector operator()( - torch::Tensor input, - int dim, - torch::optional> out - ) const { - - if (input.is_sparse()) - throw std::runtime_error("Sparse tensors are not supported"); - - if (input.device().type() != torch::DeviceType::CPU) - throw std::runtime_error("Only CPU tensors are supported"); - - if (out != torch::nullopt) - throw std::runtime_error("out argument is not supported"); - - auto in = (dim != -1) ? - torch::transpose(input, dim, -1) : - input; - - auto in_sizes = in.sizes(); - - // std::cout << "in_sizes: " << in_sizes << std::endl; - - in = in.view({ -1, in.size(-1) }).contiguous(); - - auto in_outer_stride = in.stride(-2); - auto in_inner_stride = in.stride(-1); - - auto pin = static_cast(in.data_ptr()); - - auto x = in.clone(); - - auto x_outer_stride = x.stride(-2); - auto x_inner_stride = x.stride(-1); - - auto n_cols = x.size(1); - auto n_rows = x.size(0); - auto px = static_cast(x.data_ptr()); - - auto y = torch::empty({ n_rows, n_cols }, - torch::TensorOptions().dtype(torch::kInt64)); - - auto y_outer_stride = y.stride(-2); - auto y_inner_stride = y.stride(-1); - - auto py = static_cast(y.data_ptr()); - - #pragma omp parallel for - for (decltype(n_rows) i = 0; i < n_rows; i++) { - std::vector indices(n_cols); - for (decltype(n_cols) k = 0; k < n_cols; k++) { - indices[k] = k; - } - - std::stable_sort(std::begin(indices), std::end(indices), - [pin, i, in_outer_stride, in_inner_stride](const auto &a, const auto &b) { - auto va = pin[i * in_outer_stride + a * in_inner_stride]; - auto vb = pin[i * in_outer_stride + b * in_inner_stride]; - if constexpr(descending) - return (vb < va); - else - return (va < vb); - }); - - for (decltype(n_cols) k = 0; k < n_cols; k++) { - py[i * y_outer_stride + k * y_inner_stride] = indices[k]; - px[i * x_outer_stride + k * x_inner_stride] = - pin[i * in_outer_stride + indices[k] * in_inner_stride]; - } - } - - // std::cout << "Here" << std::endl; - - x = x.view(in_sizes); - y = y.view(in_sizes); - - x = (dim == -1) ? - x : - torch::transpose(x, dim, -1).contiguous(); - - y = (dim == -1) ? - y : - torch::transpose(y, dim, -1).contiguous(); - - // std::cout << "Here 2" << std::endl; - - return { x, y }; - } -}; - -template -struct stable_sort_impl_desc: stable_sort_impl {}; - -template -struct stable_sort_impl_asc: stable_sort_impl {}; +#include "torch_stablesort_cuda.h" +#include "torch_stablesort_cpu.h" std::vector stable_sort( torch::Tensor input, @@ -110,12 +13,14 @@ std::vector stable_sort( bool descending = false, torch::optional> out = torch::nullopt) { - if (descending) - return dispatch>( - input, dim, out); - else - return dispatch>( - input, dim, out); + switch (input.device().type()) { + case torch::DeviceType::CUDA: + return dispatch_cuda(input, dim, descending, out); + case torch::DeviceType::CPU: + return dispatch_cpu(input, dim, descending, out); + default: + throw std::runtime_error("Unsupported device type"); + } } PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { diff --git a/src/torch_stablesort/torch_stablesort_cpu.h b/src/torch_stablesort/torch_stablesort_cpu.h new file mode 100644 index 0000000..b2bbce1 --- /dev/null +++ b/src/torch_stablesort/torch_stablesort_cpu.h @@ -0,0 +1,119 @@ +#pragma once + +#include + +#include +#include + +#include "dispatch.h" + +template +struct stable_sort_impl { + std::vector operator()( + torch::Tensor input, + int dim, + torch::optional> out + ) const { + + if (input.is_sparse()) + throw std::runtime_error("Sparse tensors are not supported"); + + if (input.device().type() != torch::DeviceType::CPU) + throw std::runtime_error("Only CPU tensors are supported"); + + if (out != torch::nullopt) + throw std::runtime_error("out argument is not supported"); + + auto in = (dim != -1) ? + torch::transpose(input, dim, -1) : + input; + + auto in_sizes = in.sizes(); + + // std::cout << "in_sizes: " << in_sizes << std::endl; + + in = in.view({ -1, in.size(-1) }).contiguous(); + + auto in_outer_stride = in.stride(-2); + auto in_inner_stride = in.stride(-1); + + auto pin = static_cast(in.data_ptr()); + + auto x = in.clone(); + + auto x_outer_stride = x.stride(-2); + auto x_inner_stride = x.stride(-1); + + auto n_cols = x.size(1); + auto n_rows = x.size(0); + auto px = static_cast(x.data_ptr()); + + auto y = torch::empty({ n_rows, n_cols }, + torch::TensorOptions().dtype(torch::kInt64)); + + auto y_outer_stride = y.stride(-2); + auto y_inner_stride = y.stride(-1); + + auto py = static_cast(y.data_ptr()); + + #pragma omp parallel for + for (decltype(n_rows) i = 0; i < n_rows; i++) { + std::vector indices(n_cols); + for (decltype(n_cols) k = 0; k < n_cols; k++) { + indices[k] = k; + } + + std::stable_sort(std::begin(indices), std::end(indices), + [pin, i, in_outer_stride, in_inner_stride](const auto &a, const auto &b) { + auto va = pin[i * in_outer_stride + a * in_inner_stride]; + auto vb = pin[i * in_outer_stride + b * in_inner_stride]; + if constexpr(descending) + return (vb < va); + else + return (va < vb); + }); + + for (decltype(n_cols) k = 0; k < n_cols; k++) { + py[i * y_outer_stride + k * y_inner_stride] = indices[k]; + px[i * x_outer_stride + k * x_inner_stride] = + pin[i * in_outer_stride + indices[k] * in_inner_stride]; + } + } + + // std::cout << "Here" << std::endl; + + x = x.view(in_sizes); + y = y.view(in_sizes); + + x = (dim == -1) ? + x : + torch::transpose(x, dim, -1).contiguous(); + + y = (dim == -1) ? + y : + torch::transpose(y, dim, -1).contiguous(); + + // std::cout << "Here 2" << std::endl; + + return { x, y }; + } +}; + +template +struct stable_sort_impl_desc: stable_sort_impl {}; + +template +struct stable_sort_impl_asc: stable_sort_impl {}; + +std::vector dispatch_cpu(torch::Tensor input, + int dim, + bool descending, + torch::optional> out) { + + if (descending) + return dispatch>( + input, dim, out); + else + return dispatch>( + input, dim, out); +} diff --git a/src/torch_stablesort/torch_stablesort_cuda.h b/src/torch_stablesort/torch_stablesort_cuda.h new file mode 100644 index 0000000..496429f --- /dev/null +++ b/src/torch_stablesort/torch_stablesort_cuda.h @@ -0,0 +1,109 @@ +#pragma once + +#include + +#include +#include +#include + +#include +#include + +#include "dispatch.h" + +template +struct stable_sort_impl_cuda { + std::vector operator()( + torch::Tensor input, + int dim, + torch::optional> out + ) const { + + if (input.is_sparse()) + throw std::runtime_error("Sparse tensors are not supported"); + + if (input.device().type() != torch::DeviceType::CUDA) + throw std::runtime_error("Only CUDA tensors are supported"); + + if (out != torch::nullopt) + throw std::runtime_error("out argument is not supported"); + + auto x = input.clone(); + + if (dim != -1) + x = torch::transpose(x, dim, -1); + + auto x_sizes = x.sizes(); + + x = x.view({ -1, x.size(-1) }).contiguous(); + + auto x_outer_stride = x.stride(-2); + auto x_inner_stride = x.stride(-1); + auto n_cols = x.size(1); + auto n_rows = x.size(0); + auto px = x.data_ptr(); + + assert(x_inner_stride == 1); + + auto y = torch::repeat_interleave( + torch::arange(0, n_cols, 1, torch::TensorOptions() + .dtype(torch::kInt32) + .device(x.device())), + torch::ones(n_rows, torch::TensorOptions() + .dtype(torch::kInt32) + .device(x.device())) + ); + + auto y_outer_stride = y.stride(-2); + auto y_inner_stride = y.stride(-1); + auto py = y.data_ptr(); + + assert(y_inner_stride == 1); + + for (decltype(n_rows) i = 0; i < n_rows; i++) { + auto ind_beg = thrust::device_pointer_cast(py + i * y_outer_stride); + + auto val_beg = thrust::device_pointer_cast(px + i * x_outer_stride); + auto val_end = thrust::device_pointer_cast(px + i * x_outer_stride + + n_cols * x_inner_stride); + + if constexpr(descending) + thrust::stable_sort_by_key(thrust::device, val_beg, val_end, ind_beg, + thrust::greater()); + else + thrust::stable_sort_by_key(thrust::device, val_beg, val_end, ind_beg); + } + + x = x.view(x_sizes); + y = y.view(x_sizes); + + x = (dim == -1) ? + x : + torch::transpose(x, dim, -1).contiguous(); + + y = (dim == -1) ? + y : + torch::transpose(y, dim, -1).contiguous(); + + return { x, y }; + } +}; + +template +struct stable_sort_impl_desc_cuda: stable_sort_impl_cuda {}; + +template +struct stable_sort_impl_asc_cuda: stable_sort_impl_cuda {}; + +std::vector dispatch_cuda(torch::Tensor input, + int dim, + bool descending, + torch::optional> out) { + + if (descending) + return dispatch>( + input, dim, out); + else + return dispatch>( + input, dim, out); +} -- 2.26.2 From 62af8fc5f16bccbccb6086a19e53c6d6c0f5a777 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 1 Sep 2020 11:24:07 +0200 Subject: [PATCH 224/227] Progress with CUDA version of torch_stablesort. --- src/torch_stablesort/dispatch.h | 1 + src/torch_stablesort/setup.py | 13 +- src/torch_stablesort/torch_stablesort_cpu.cpp | 115 ++++++++++++++++++ src/torch_stablesort/torch_stablesort_cpu.h | 110 +---------------- src/torch_stablesort/torch_stablesort_cuda.cu | 105 ++++++++++++++++ src/torch_stablesort/torch_stablesort_cuda.h | 100 +-------------- 6 files changed, 231 insertions(+), 213 deletions(-) create mode 100644 src/torch_stablesort/torch_stablesort_cpu.cpp create mode 100644 src/torch_stablesort/torch_stablesort_cuda.cu diff --git a/src/torch_stablesort/dispatch.h b/src/torch_stablesort/dispatch.h index a7dfb59..ef91b46 100644 --- a/src/torch_stablesort/dispatch.h +++ b/src/torch_stablesort/dispatch.h @@ -1,5 +1,6 @@ #pragma once +#include #include template class F, typename R, typename... Ts> diff --git a/src/torch_stablesort/setup.py b/src/torch_stablesort/setup.py index 8001773..ab9412c 100644 --- a/src/torch_stablesort/setup.py +++ b/src/torch_stablesort/setup.py @@ -3,8 +3,11 @@ from torch.utils import cpp_extension setup(name='torch_stablesort', py_modules=['torch_stablesort'], - ext_modules=[cpp_extension.CppExtension('torch_stablesort_cpp', - ['torch_stablesort.cpp'], - extra_compile_args=['-I/pstore/home/adaszews/scratch/thrust', - '-fopenmp', '-ggdb', '-std=c++1z'])], - cmdclass={'build_ext': cpp_extension.BuildExtension}) + ext_modules=[ cpp_extension.CUDAExtension( 'torch_stablesort_cpp', + ['torch_stablesort.cpp', 'torch_stablesort_cpu.cpp', 'torch_stablesort_cuda.cu'], + extra_compile_args={ + 'cxx': ['-fopenmp', '-ggdb', '-std=c++1z'], + 'nvcc': [ '-I/pstore/home/adaszews/scratch/thrust', + '-ccbin', '/pstore/data/data_science/app/modules/anaconda3-2020.07/bin/x86_64-conda_cos6-linux-gnu-gcc', '-std=c++14'] + } ) ], + cmdclass={'build_ext': cpp_extension.BuildExtension}) diff --git a/src/torch_stablesort/torch_stablesort_cpu.cpp b/src/torch_stablesort/torch_stablesort_cpu.cpp new file mode 100644 index 0000000..12cef54 --- /dev/null +++ b/src/torch_stablesort/torch_stablesort_cpu.cpp @@ -0,0 +1,115 @@ +#include + +#include "dispatch.h" +#include "torch_stablesort_cpu.h" + +template +struct stable_sort_impl { + std::vector operator()( + torch::Tensor input, + int dim, + torch::optional> out + ) const { + + if (input.is_sparse()) + throw std::runtime_error("Sparse tensors are not supported"); + + if (input.device().type() != torch::DeviceType::CPU) + throw std::runtime_error("Only CPU tensors are supported"); + + if (out != torch::nullopt) + throw std::runtime_error("out argument is not supported"); + + auto in = (dim != -1) ? + torch::transpose(input, dim, -1) : + input; + + auto in_sizes = in.sizes(); + + // std::cout << "in_sizes: " << in_sizes << std::endl; + + in = in.view({ -1, in.size(-1) }).contiguous(); + + auto in_outer_stride = in.stride(-2); + auto in_inner_stride = in.stride(-1); + + auto pin = static_cast(in.data_ptr()); + + auto x = in.clone(); + + auto x_outer_stride = x.stride(-2); + auto x_inner_stride = x.stride(-1); + + auto n_cols = x.size(1); + auto n_rows = x.size(0); + auto px = static_cast(x.data_ptr()); + + auto y = torch::empty({ n_rows, n_cols }, + torch::TensorOptions().dtype(torch::kInt64)); + + auto y_outer_stride = y.stride(-2); + auto y_inner_stride = y.stride(-1); + + auto py = static_cast(y.data_ptr()); + + #pragma omp parallel for + for (decltype(n_rows) i = 0; i < n_rows; i++) { + std::vector indices(n_cols); + for (decltype(n_cols) k = 0; k < n_cols; k++) { + indices[k] = k; + } + + std::stable_sort(std::begin(indices), std::end(indices), + [pin, i, in_outer_stride, in_inner_stride](const auto &a, const auto &b) { + auto va = pin[i * in_outer_stride + a * in_inner_stride]; + auto vb = pin[i * in_outer_stride + b * in_inner_stride]; + if constexpr(descending) + return (vb < va); + else + return (va < vb); + }); + + for (decltype(n_cols) k = 0; k < n_cols; k++) { + py[i * y_outer_stride + k * y_inner_stride] = indices[k]; + px[i * x_outer_stride + k * x_inner_stride] = + pin[i * in_outer_stride + indices[k] * in_inner_stride]; + } + } + + // std::cout << "Here" << std::endl; + + x = x.view(in_sizes); + y = y.view(in_sizes); + + x = (dim == -1) ? + x : + torch::transpose(x, dim, -1).contiguous(); + + y = (dim == -1) ? + y : + torch::transpose(y, dim, -1).contiguous(); + + // std::cout << "Here 2" << std::endl; + + return { x, y }; + } +}; + +template +struct stable_sort_impl_desc: stable_sort_impl {}; + +template +struct stable_sort_impl_asc: stable_sort_impl {}; + +std::vector dispatch_cpu(torch::Tensor input, + int dim, + bool descending, + torch::optional> out) { + + if (descending) + return dispatch>( + input, dim, out); + else + return dispatch>( + input, dim, out); +} diff --git a/src/torch_stablesort/torch_stablesort_cpu.h b/src/torch_stablesort/torch_stablesort_cpu.h index b2bbce1..7b54251 100644 --- a/src/torch_stablesort/torch_stablesort_cpu.h +++ b/src/torch_stablesort/torch_stablesort_cpu.h @@ -5,115 +5,7 @@ #include #include -#include "dispatch.h" - -template -struct stable_sort_impl { - std::vector operator()( - torch::Tensor input, - int dim, - torch::optional> out - ) const { - - if (input.is_sparse()) - throw std::runtime_error("Sparse tensors are not supported"); - - if (input.device().type() != torch::DeviceType::CPU) - throw std::runtime_error("Only CPU tensors are supported"); - - if (out != torch::nullopt) - throw std::runtime_error("out argument is not supported"); - - auto in = (dim != -1) ? - torch::transpose(input, dim, -1) : - input; - - auto in_sizes = in.sizes(); - - // std::cout << "in_sizes: " << in_sizes << std::endl; - - in = in.view({ -1, in.size(-1) }).contiguous(); - - auto in_outer_stride = in.stride(-2); - auto in_inner_stride = in.stride(-1); - - auto pin = static_cast(in.data_ptr()); - - auto x = in.clone(); - - auto x_outer_stride = x.stride(-2); - auto x_inner_stride = x.stride(-1); - - auto n_cols = x.size(1); - auto n_rows = x.size(0); - auto px = static_cast(x.data_ptr()); - - auto y = torch::empty({ n_rows, n_cols }, - torch::TensorOptions().dtype(torch::kInt64)); - - auto y_outer_stride = y.stride(-2); - auto y_inner_stride = y.stride(-1); - - auto py = static_cast(y.data_ptr()); - - #pragma omp parallel for - for (decltype(n_rows) i = 0; i < n_rows; i++) { - std::vector indices(n_cols); - for (decltype(n_cols) k = 0; k < n_cols; k++) { - indices[k] = k; - } - - std::stable_sort(std::begin(indices), std::end(indices), - [pin, i, in_outer_stride, in_inner_stride](const auto &a, const auto &b) { - auto va = pin[i * in_outer_stride + a * in_inner_stride]; - auto vb = pin[i * in_outer_stride + b * in_inner_stride]; - if constexpr(descending) - return (vb < va); - else - return (va < vb); - }); - - for (decltype(n_cols) k = 0; k < n_cols; k++) { - py[i * y_outer_stride + k * y_inner_stride] = indices[k]; - px[i * x_outer_stride + k * x_inner_stride] = - pin[i * in_outer_stride + indices[k] * in_inner_stride]; - } - } - - // std::cout << "Here" << std::endl; - - x = x.view(in_sizes); - y = y.view(in_sizes); - - x = (dim == -1) ? - x : - torch::transpose(x, dim, -1).contiguous(); - - y = (dim == -1) ? - y : - torch::transpose(y, dim, -1).contiguous(); - - // std::cout << "Here 2" << std::endl; - - return { x, y }; - } -}; - -template -struct stable_sort_impl_desc: stable_sort_impl {}; - -template -struct stable_sort_impl_asc: stable_sort_impl {}; - std::vector dispatch_cpu(torch::Tensor input, int dim, bool descending, - torch::optional> out) { - - if (descending) - return dispatch>( - input, dim, out); - else - return dispatch>( - input, dim, out); -} + torch::optional> out); diff --git a/src/torch_stablesort/torch_stablesort_cuda.cu b/src/torch_stablesort/torch_stablesort_cuda.cu new file mode 100644 index 0000000..bafa9ee --- /dev/null +++ b/src/torch_stablesort/torch_stablesort_cuda.cu @@ -0,0 +1,105 @@ +#pragma once + +#include +#include +#include + +#include "dispatch.h" +#include "torch_stablesort_cuda.h" + +template +struct stable_sort_impl_cuda { + std::vector operator()( + torch::Tensor input, + int dim, + torch::optional> out + ) const { + + if (input.is_sparse()) + throw std::runtime_error("Sparse tensors are not supported"); + + if (input.device().type() != torch::DeviceType::CUDA) + throw std::runtime_error("Only CUDA tensors are supported"); + + if (out != torch::nullopt) + throw std::runtime_error("out argument is not supported"); + + auto x = input.clone(); + + if (dim != -1) + x = torch::transpose(x, dim, -1); + + auto x_sizes = x.sizes(); + + x = x.view({ -1, x.size(-1) }).contiguous(); + + auto x_outer_stride = x.stride(-2); + auto x_inner_stride = x.stride(-1); + auto n_cols = x.size(1); + auto n_rows = x.size(0); + auto px = x.data_ptr(); + + assert(x_inner_stride == 1); + + auto y = torch::repeat_interleave( + torch::arange(0, n_cols, 1, torch::TensorOptions() + .dtype(torch::kInt32) + .device(x.device())), + torch::ones(n_rows, torch::TensorOptions() + .dtype(torch::kInt32) + .device(x.device())) + ); + + auto y_outer_stride = y.stride(-2); + auto y_inner_stride = y.stride(-1); + auto py = y.data_ptr(); + + assert(y_inner_stride == 1); + + for (decltype(n_rows) i = 0; i < n_rows; i++) { + auto ind_beg = thrust::device_pointer_cast(py + i * y_outer_stride); + + auto val_beg = thrust::device_pointer_cast(px + i * x_outer_stride); + auto val_end = thrust::device_pointer_cast(px + i * x_outer_stride + + n_cols * x_inner_stride); + + if constexpr(descending) + thrust::stable_sort_by_key(thrust::device, val_beg, val_end, ind_beg, + thrust::greater()); + else + thrust::stable_sort_by_key(thrust::device, val_beg, val_end, ind_beg); + } + + x = x.view(x_sizes); + y = y.view(x_sizes); + + x = (dim == -1) ? + x : + torch::transpose(x, dim, -1).contiguous(); + + y = (dim == -1) ? + y : + torch::transpose(y, dim, -1).contiguous(); + + return { x, y }; + } +}; + +template +struct stable_sort_impl_desc_cuda: stable_sort_impl_cuda {}; + +template +struct stable_sort_impl_asc_cuda: stable_sort_impl_cuda {}; + +std::vector dispatch_cuda(torch::Tensor input, + int dim, + bool descending, + torch::optional> out) { + + if (descending) + return dispatch>( + input, dim, out); + else + return dispatch>( + input, dim, out); +} diff --git a/src/torch_stablesort/torch_stablesort_cuda.h b/src/torch_stablesort/torch_stablesort_cuda.h index 496429f..e5f10e4 100644 --- a/src/torch_stablesort/torch_stablesort_cuda.h +++ b/src/torch_stablesort/torch_stablesort_cuda.h @@ -2,108 +2,10 @@ #include -#include -#include -#include - #include #include -#include "dispatch.h" - -template -struct stable_sort_impl_cuda { - std::vector operator()( - torch::Tensor input, - int dim, - torch::optional> out - ) const { - - if (input.is_sparse()) - throw std::runtime_error("Sparse tensors are not supported"); - - if (input.device().type() != torch::DeviceType::CUDA) - throw std::runtime_error("Only CUDA tensors are supported"); - - if (out != torch::nullopt) - throw std::runtime_error("out argument is not supported"); - - auto x = input.clone(); - - if (dim != -1) - x = torch::transpose(x, dim, -1); - - auto x_sizes = x.sizes(); - - x = x.view({ -1, x.size(-1) }).contiguous(); - - auto x_outer_stride = x.stride(-2); - auto x_inner_stride = x.stride(-1); - auto n_cols = x.size(1); - auto n_rows = x.size(0); - auto px = x.data_ptr(); - - assert(x_inner_stride == 1); - - auto y = torch::repeat_interleave( - torch::arange(0, n_cols, 1, torch::TensorOptions() - .dtype(torch::kInt32) - .device(x.device())), - torch::ones(n_rows, torch::TensorOptions() - .dtype(torch::kInt32) - .device(x.device())) - ); - - auto y_outer_stride = y.stride(-2); - auto y_inner_stride = y.stride(-1); - auto py = y.data_ptr(); - - assert(y_inner_stride == 1); - - for (decltype(n_rows) i = 0; i < n_rows; i++) { - auto ind_beg = thrust::device_pointer_cast(py + i * y_outer_stride); - - auto val_beg = thrust::device_pointer_cast(px + i * x_outer_stride); - auto val_end = thrust::device_pointer_cast(px + i * x_outer_stride + - n_cols * x_inner_stride); - - if constexpr(descending) - thrust::stable_sort_by_key(thrust::device, val_beg, val_end, ind_beg, - thrust::greater()); - else - thrust::stable_sort_by_key(thrust::device, val_beg, val_end, ind_beg); - } - - x = x.view(x_sizes); - y = y.view(x_sizes); - - x = (dim == -1) ? - x : - torch::transpose(x, dim, -1).contiguous(); - - y = (dim == -1) ? - y : - torch::transpose(y, dim, -1).contiguous(); - - return { x, y }; - } -}; - -template -struct stable_sort_impl_desc_cuda: stable_sort_impl_cuda {}; - -template -struct stable_sort_impl_asc_cuda: stable_sort_impl_cuda {}; - std::vector dispatch_cuda(torch::Tensor input, int dim, bool descending, - torch::optional> out) { - - if (descending) - return dispatch>( - input, dim, out); - else - return dispatch>( - input, dim, out); -} + torch::optional> out); -- 2.26.2 From c1353168f313e754eafe6b9f66a5f1ffb5e055b5 Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 1 Sep 2020 13:08:14 +0200 Subject: [PATCH 225/227] CUDA performance needs improvement. --- src/torch_stablesort/setup.py | 3 +- src/torch_stablesort/torch_stablesort_cuda.cu | 49 ++++++++++++------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/torch_stablesort/setup.py b/src/torch_stablesort/setup.py index ab9412c..ae03d78 100644 --- a/src/torch_stablesort/setup.py +++ b/src/torch_stablesort/setup.py @@ -8,6 +8,7 @@ setup(name='torch_stablesort', extra_compile_args={ 'cxx': ['-fopenmp', '-ggdb', '-std=c++1z'], 'nvcc': [ '-I/pstore/home/adaszews/scratch/thrust', - '-ccbin', '/pstore/data/data_science/app/modules/anaconda3-2020.07/bin/x86_64-conda_cos6-linux-gnu-gcc', '-std=c++14'] + '-ccbin', '/pstore/data/data_science/app/modules/anaconda3-2020.07/bin/x86_64-conda_cos6-linux-gnu-gcc', + '-std=c++14', '--expt-extended-lambda', '-O99'] } ) ], cmdclass={'build_ext': cpp_extension.BuildExtension}) diff --git a/src/torch_stablesort/torch_stablesort_cuda.cu b/src/torch_stablesort/torch_stablesort_cuda.cu index bafa9ee..d7885d7 100644 --- a/src/torch_stablesort/torch_stablesort_cuda.cu +++ b/src/torch_stablesort/torch_stablesort_cuda.cu @@ -43,32 +43,43 @@ struct stable_sort_impl_cuda { auto y = torch::repeat_interleave( torch::arange(0, n_cols, 1, torch::TensorOptions() - .dtype(torch::kInt32) - .device(x.device())), - torch::ones(n_rows, torch::TensorOptions() - .dtype(torch::kInt32) - .device(x.device())) + .dtype(torch::kInt64) + .device(x.device())).view({ 1, -1 }), + n_rows, + 0 /* dim */ ); auto y_outer_stride = y.stride(-2); auto y_inner_stride = y.stride(-1); - auto py = y.data_ptr(); + auto py = y.data_ptr(); assert(y_inner_stride == 1); - for (decltype(n_rows) i = 0; i < n_rows; i++) { - auto ind_beg = thrust::device_pointer_cast(py + i * y_outer_stride); - - auto val_beg = thrust::device_pointer_cast(px + i * x_outer_stride); - auto val_end = thrust::device_pointer_cast(px + i * x_outer_stride + - n_cols * x_inner_stride); - - if constexpr(descending) - thrust::stable_sort_by_key(thrust::device, val_beg, val_end, ind_beg, - thrust::greater()); - else - thrust::stable_sort_by_key(thrust::device, val_beg, val_end, ind_beg); - } + #define NUM_STREAMS 16 + cudaStream_t streams[NUM_STREAMS]; + for(int i = 0; i < NUM_STREAMS; i++) + assert(cudaStreamCreate(&streams[i]) == cudaSuccess); + + thrust::host_vector row_indices(n_rows); + thrust::sequence(row_indices.begin(), row_indices.end()); + thrust::for_each(thrust::host, row_indices.begin(), row_indices.end(), + [&streams, py, y_outer_stride, px, x_outer_stride, x_inner_stride, n_cols](int64_t i) { + auto ind_beg = thrust::device_pointer_cast(py + i * y_outer_stride); + + auto val_beg = thrust::device_pointer_cast(px + i * x_outer_stride); + auto val_end = thrust::device_pointer_cast(px + i * x_outer_stride + + n_cols * x_inner_stride); + + if (descending) + thrust::stable_sort_by_key(thrust::cuda::par.on(streams[i % NUM_STREAMS]), val_beg, val_end, ind_beg, + thrust::greater()); + else + thrust::stable_sort_by_key(thrust::cuda::par.on(streams[i % NUM_STREAMS]), val_beg, val_end, ind_beg); + }); + cudaDeviceSynchronize(); + + for(int i = 0; i < NUM_STREAMS; i++) + assert(cudaStreamDestroy(streams[i]) == cudaSuccess); x = x.view(x_sizes); y = y.view(x_sizes); -- 2.26.2 From 897a6f0722c7eb479f16d275dd4a62b1a85d275a Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 1 Sep 2020 18:10:36 +0200 Subject: [PATCH 226/227] torch_stablesort_cuda.cu getting there... --- src/torch_stablesort/torch_stablesort_cuda.cu | 100 +++++++++--------- 1 file changed, 49 insertions(+), 51 deletions(-) diff --git a/src/torch_stablesort/torch_stablesort_cuda.cu b/src/torch_stablesort/torch_stablesort_cuda.cu index d7885d7..e05aa2c 100644 --- a/src/torch_stablesort/torch_stablesort_cuda.cu +++ b/src/torch_stablesort/torch_stablesort_cuda.cu @@ -24,75 +24,73 @@ struct stable_sort_impl_cuda { if (out != torch::nullopt) throw std::runtime_error("out argument is not supported"); - auto x = input.clone(); + auto values = input.clone(); if (dim != -1) - x = torch::transpose(x, dim, -1); + values = torch::transpose(values, dim, -1); - auto x_sizes = x.sizes(); + auto values_sizes = values.sizes(); - x = x.view({ -1, x.size(-1) }).contiguous(); + values = values.view({ -1, values.size(-1) }).contiguous(); - auto x_outer_stride = x.stride(-2); - auto x_inner_stride = x.stride(-1); - auto n_cols = x.size(1); - auto n_rows = x.size(0); - auto px = x.data_ptr(); + auto n_cols = values.size(1); + auto n_rows = values.size(0); - assert(x_inner_stride == 1); + assert(values.stride(-2) == n_cols); + assert(values.stride(-1) == 1); - auto y = torch::repeat_interleave( + auto values_ptr = values.data_ptr(); + + auto indices = torch::repeat_interleave( torch::arange(0, n_cols, 1, torch::TensorOptions() .dtype(torch::kInt64) - .device(x.device())).view({ 1, -1 }), + .device(values.device())).view({ 1, -1 }), n_rows, 0 /* dim */ ); - auto y_outer_stride = y.stride(-2); - auto y_inner_stride = y.stride(-1); - auto py = y.data_ptr(); - - assert(y_inner_stride == 1); - - #define NUM_STREAMS 16 - cudaStream_t streams[NUM_STREAMS]; - for(int i = 0; i < NUM_STREAMS; i++) - assert(cudaStreamCreate(&streams[i]) == cudaSuccess); - - thrust::host_vector row_indices(n_rows); - thrust::sequence(row_indices.begin(), row_indices.end()); - thrust::for_each(thrust::host, row_indices.begin(), row_indices.end(), - [&streams, py, y_outer_stride, px, x_outer_stride, x_inner_stride, n_cols](int64_t i) { - auto ind_beg = thrust::device_pointer_cast(py + i * y_outer_stride); - - auto val_beg = thrust::device_pointer_cast(px + i * x_outer_stride); - auto val_end = thrust::device_pointer_cast(px + i * x_outer_stride + - n_cols * x_inner_stride); - - if (descending) - thrust::stable_sort_by_key(thrust::cuda::par.on(streams[i % NUM_STREAMS]), val_beg, val_end, ind_beg, - thrust::greater()); - else - thrust::stable_sort_by_key(thrust::cuda::par.on(streams[i % NUM_STREAMS]), val_beg, val_end, ind_beg); - }); - cudaDeviceSynchronize(); + assert(indices.stride(-2) == n_cols); + assert(indices.stride(-1) == 1); + auto indices_ptr = indices.data_ptr(); + + auto n = n_rows * n_cols; + + auto ind_beg = thrust::device_pointer_cast(indices_ptr); + auto val_beg = thrust::device_pointer_cast(values_ptr); + + if (descending) + thrust::stable_sort_by_key(thrust::device, val_beg, val_beg + n, ind_beg, thrust::greater()); + else + thrust::stable_sort_by_key(thrust::device, val_beg, val_beg + n, ind_beg); + + thrust::device_vector segments(n); + thrust::constant_iterator n_cols_iter(n_cols); + thrust::transform(thrust::device, + ind_beg, ind_beg + n, n_cols_iter, + segments.begin(), thrust::divides()); - for(int i = 0; i < NUM_STREAMS; i++) - assert(cudaStreamDestroy(streams[i]) == cudaSuccess); + thrust::stable_sort_by_key(thrust::device, segments.begin(), + segments.end(), val_beg); - x = x.view(x_sizes); - y = y.view(x_sizes); + thrust::transform(thrust::device, + ind_beg, ind_beg + n, n_cols_iter, + segments.begin(), thrust::modulus()); - x = (dim == -1) ? - x : - torch::transpose(x, dim, -1).contiguous(); + thrust::stable_sort_by_key(thrust::device, segments.begin(), + segments.end(), ind_beg); - y = (dim == -1) ? - y : - torch::transpose(y, dim, -1).contiguous(); + cudaDeviceSynchronize(); + + values = values.view(values_sizes); + indices = indices.view(values_sizes); + + if (dim != -1) + values = torch::transpose(values, dim, -1).contiguous(); + + if (dim != -1) + indices = torch::transpose(indices, dim, -1).contiguous(); - return { x, y }; + return { values, indices }; } }; -- 2.26.2 From 5cf1e3d1b594cdbeff88c9e97268449e05089cac Mon Sep 17 00:00:00 2001 From: Stanislaw Adaszewski Date: Tue, 1 Sep 2020 18:32:58 +0200 Subject: [PATCH 227/227] torch_stablesort CUDA version works and is fast. --- src/torch_stablesort/torch_stablesort_cuda.cu | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/torch_stablesort/torch_stablesort_cuda.cu b/src/torch_stablesort/torch_stablesort_cuda.cu index e05aa2c..cf9b0d7 100644 --- a/src/torch_stablesort/torch_stablesort_cuda.cu +++ b/src/torch_stablesort/torch_stablesort_cuda.cu @@ -29,31 +29,27 @@ struct stable_sort_impl_cuda { if (dim != -1) values = torch::transpose(values, dim, -1); - auto values_sizes = values.sizes(); + auto orig_sizes = values.sizes(); values = values.view({ -1, values.size(-1) }).contiguous(); auto n_cols = values.size(1); auto n_rows = values.size(0); + auto n = n_rows * n_cols; assert(values.stride(-2) == n_cols); assert(values.stride(-1) == 1); auto values_ptr = values.data_ptr(); - auto indices = torch::repeat_interleave( - torch::arange(0, n_cols, 1, torch::TensorOptions() + auto indices = torch::arange(0, n, 1, torch::TensorOptions() .dtype(torch::kInt64) - .device(values.device())).view({ 1, -1 }), - n_rows, - 0 /* dim */ - ); + .device(values.device())).view({ n_rows, n_cols }); assert(indices.stride(-2) == n_cols); assert(indices.stride(-1) == 1); - auto indices_ptr = indices.data_ptr(); - auto n = n_rows * n_cols; + auto indices_ptr = indices.data_ptr(); auto ind_beg = thrust::device_pointer_cast(indices_ptr); auto val_beg = thrust::device_pointer_cast(values_ptr); @@ -74,15 +70,18 @@ struct stable_sort_impl_cuda { thrust::transform(thrust::device, ind_beg, ind_beg + n, n_cols_iter, - segments.begin(), thrust::modulus()); + segments.begin(), thrust::divides()); thrust::stable_sort_by_key(thrust::device, segments.begin(), segments.end(), ind_beg); + thrust::transform(thrust::device, ind_beg, ind_beg + n, + n_cols_iter, ind_beg, thrust::modulus()); + cudaDeviceSynchronize(); - values = values.view(values_sizes); - indices = indices.view(values_sizes); + values = values.view(orig_sizes); + indices = indices.view(orig_sizes); if (dim != -1) values = torch::transpose(values, dim, -1).contiguous(); -- 2.26.2